diff --git a/src/bindings/python/src/openvino/runtime/utils/data_helpers/data_dispatcher.py b/src/bindings/python/src/openvino/runtime/utils/data_helpers/data_dispatcher.py index 95490cbf98acb4..62beadea9d363d 100644 --- a/src/bindings/python/src/openvino/runtime/utils/data_helpers/data_dispatcher.py +++ b/src/bindings/python/src/openvino/runtime/utils/data_helpers/data_dispatcher.py @@ -56,18 +56,25 @@ def _( is_shared: bool = False, key: Optional[ValidKeys] = None, ) -> Tensor: - # Edge-case for numpy arrays if shape is "empty", - # assume this is a scalar value - always copy - if not value.shape: - return Tensor(np.ndarray([], value.dtype, np.array(value))) - tensor_type = get_request_tensor(request, key).get_element_type() + tensor = get_request_tensor(request, key) + tensor_type = tensor.get_element_type() tensor_dtype = tensor_type.to_dtype() - # WA for FP16-->BF16 edge-case - always copy + if value.ndim == 0: + tensor_shape = tuple(tensor.shape) + if tensor_dtype == value.dtype and tensor_shape == value.shape: + return Tensor(value, shared_memory=is_shared) + else: + return Tensor(value.astype(tensor_dtype).reshape(tensor_shape), shared_memory=False) + # WA for FP16-->BF16 edge-case, always copy. if tensor_type == Type.bf16: tensor = Tensor(tensor_type, value.shape) tensor.data[:] = value.view(tensor_dtype) return tensor - return Tensor(value.astype(tensor_dtype) if tensor_dtype != value.dtype else value, shared_memory=is_shared) + # If types are mismatched, convert and always copy. + if tensor_dtype != value.dtype: + return Tensor(value.astype(tensor_dtype), shared_memory=False) + # Otherwise, use mode defined in the call. + return Tensor(value, shared_memory=is_shared) @value_to_tensor.register(np.number) @@ -75,11 +82,18 @@ def _( @value_to_tensor.register(float) def _( value: ScalarTypes, - request: Optional[_InferRequestWrapper] = None, + request: _InferRequestWrapper, is_shared: bool = False, key: Optional[ValidKeys] = None, ) -> Tensor: - return Tensor(np.ndarray([], type(value), np.array(value))) + # np.number/int/float edge-case, copy will occur in both scenarios. + tensor_type = get_request_tensor(request, key).get_element_type() + tensor_dtype = tensor_type.to_dtype() + tmp = np.array(value) + # If types are mismatched, convert. + if tensor_dtype != tmp.dtype: + return Tensor(tmp.astype(tensor_dtype), shared_memory=False) + return Tensor(tmp, shared_memory=False) def to_c_style(value: Any, is_shared: bool = False) -> Any: @@ -87,7 +101,6 @@ def to_c_style(value: Any, is_shared: bool = False) -> Any: if hasattr(value, "__array__"): return to_c_style(np.array(value, copy=False)) if is_shared else np.array(value, copy=True) return value - # Check C-style if not convert data (or raise error?) return value if value.flags["C_CONTIGUOUS"] else np.ascontiguousarray(value) @@ -223,20 +236,20 @@ def _( request: _InferRequestWrapper, key: Optional[ValidKeys] = None, ) -> None: - # If shape is "empty", assume this is a scalar value - if not inputs.shape: - set_request_tensor( - request, - value_to_tensor(inputs, request=request, is_shared=False), - key, - ) - else: + if inputs.ndim != 0: tensor = get_request_tensor(request, key) # Update shape if there is a mismatch - if tensor.shape != inputs.shape: + if tuple(tensor.shape) != inputs.shape: tensor.shape = inputs.shape # When copying, type should be up/down-casted automatically. tensor.data[:] = inputs[:] + else: + # If shape is "empty", assume this is a scalar value + set_request_tensor( + request, + value_to_tensor(inputs, request=request, is_shared=False, key=key), + key, + ) @update_tensor.register(np.number) # type: ignore @@ -249,7 +262,7 @@ def _( ) -> None: set_request_tensor( request, - value_to_tensor(inputs, is_shared=False), + value_to_tensor(inputs, request=request, is_shared=False, key=key), key, ) @@ -320,7 +333,7 @@ def _( inputs: Union[Tensor, ScalarTypes], request: _InferRequestWrapper, ) -> Tensor: - return value_to_tensor(inputs, is_shared=False) + return value_to_tensor(inputs, request=request, is_shared=False) ### # End of "copied" dispatcher methods. ### diff --git a/src/bindings/python/src/pyopenvino/core/common.cpp b/src/bindings/python/src/pyopenvino/core/common.cpp index d94d99f3d9b953..2ad7e395a92895 100644 --- a/src/bindings/python/src/pyopenvino/core/common.cpp +++ b/src/bindings/python/src/pyopenvino/core/common.cpp @@ -121,7 +121,7 @@ ov::op::v0::Constant create_copied(py::array& array) { // Create actual Constant and a constructor is copying data. return ov::op::v0::Constant(array_helpers::get_ov_type(array), array_helpers::get_shape(array), - const_cast(array.data(0))); + array.ndim() == 0 ? array.data() : array.data(0)); } template <> @@ -135,10 +135,10 @@ ov::op::v0::Constant create_shared(py::array& array) { // Check if passed array has C-style contiguous memory layout. // If memory is going to be shared it needs to be contiguous before passing to the constructor. if (array_helpers::is_contiguous(array)) { - auto memory = - std::make_shared>(static_cast(array.mutable_data(0)), - array.nbytes(), - array); + auto memory = std::make_shared>( + static_cast(array.ndim() == 0 ? array.mutable_data() : array.mutable_data(0)), + array.ndim() == 0 ? array.itemsize() : array.nbytes(), + array); return ov::op::v0::Constant(array_helpers::get_ov_type(array), array_helpers::get_shape(array), memory); } // If passed array is not C-style, throw an error. @@ -159,9 +159,9 @@ ov::Tensor create_copied(py::array& array) { // Create actual Tensor and copy data. auto tensor = ov::Tensor(array_helpers::get_ov_type(array), array_helpers::get_shape(array)); // If ndim of py::array is 0, array is a numpy scalar. That results in size to be equal to 0. - // To gain access to actual raw/low-level data, it is needed to use buffer protocol. - py::buffer_info buf = array.request(); - std::memcpy(tensor.data(), buf.ptr, buf.ndim == 0 ? buf.itemsize : buf.itemsize * buf.size); + std::memcpy(tensor.data(), + array.ndim() == 0 ? array.data() : array.data(0), + array.ndim() == 0 ? array.itemsize() : array.nbytes()); return tensor; } @@ -170,9 +170,10 @@ ov::Tensor create_shared(py::array& array) { // Check if passed array has C-style contiguous memory layout. // If memory is going to be shared it needs to be contiguous before passing to the constructor. if (array_helpers::is_contiguous(array)) { + // If ndim of py::array is 0, array is a numpy scalar. return ov::Tensor(array_helpers::get_ov_type(array), array_helpers::get_shape(array), - const_cast(array.data(0)), + array.ndim() == 0 ? array.mutable_data() : array.mutable_data(0), array_helpers::get_strides(array)); } // If passed array is not C-style, throw an error. diff --git a/src/bindings/python/tests/test_runtime/test_infer_request.py b/src/bindings/python/tests/test_runtime/test_infer_request.py index b64623a4ad7c8d..971208b8a75cd5 100644 --- a/src/bindings/python/tests/test_runtime/test_infer_request.py +++ b/src/bindings/python/tests/test_runtime/test_infer_request.py @@ -1014,3 +1014,108 @@ def test_convert_infer_request(device): with pytest.raises(TypeError) as e: deepcopy(res) assert "cannot deepcopy 'openvino.runtime.ConstOutput' object." in str(e) + + +@pytest.mark.parametrize("shared_flag", [True, False]) +@pytest.mark.parametrize("input_data", [ + np.array(1.0, dtype=np.float32), + np.array(1, dtype=np.int32), + np.float32(1.0), + np.int32(1.0), + 1.0, + 1, +]) +def test_only_scalar_infer(device, shared_flag, input_data): + core = Core() + param = ops.parameter([], np.float32, name="data") + relu = ops.relu(param, name="relu") + model = Model([relu], [param], "scalar_model") + + compiled = core.compile_model(model=model, device_name=device) + request = compiled.create_infer_request() + + res = request.infer(input_data, shared_memory=shared_flag) + + assert res[request.model_outputs[0]] == np.maximum(input_data, 0) + + input_tensor = request.get_input_tensor() + if shared_flag and isinstance(input_data, np.ndarray) and input_data.dtype == input_tensor.data.dtype: + assert np.shares_memory(input_data, input_tensor.data) + else: + assert not np.shares_memory(input_data, input_tensor.data) + + +@pytest.mark.parametrize("shared_flag", [True, False]) +@pytest.mark.parametrize("input_data", [ + {0: np.array(1.0, dtype=np.float32), 1: np.array([1.0, 2.0], dtype=np.float32)}, + {0: np.array(1, dtype=np.int32), 1: np.array([1, 2], dtype=np.int32)}, + {0: np.float32(1.0), 1: np.array([1, 2], dtype=np.float32)}, + {0: np.int32(1.0), 1: np.array([1, 2], dtype=np.int32)}, + {0: 1.0, 1: np.array([1.0, 2.0], dtype=np.float32)}, + {0: 1, 1: np.array([1.0, 2.0], dtype=np.int32)}, +]) +def test_mixed_scalar_infer(device, shared_flag, input_data): + core = Core() + param0 = ops.parameter([], np.float32, name="data0") + param1 = ops.parameter([2], np.float32, name="data1") + add = ops.add(param0, param1, name="add") + model = Model([add], [param0, param1], "mixed_model") + + compiled = core.compile_model(model=model, device_name=device) + request = compiled.create_infer_request() + + res = request.infer(input_data, shared_memory=shared_flag) + + assert np.allclose(res[request.model_outputs[0]], np.add(input_data[0], input_data[1])) + + input_tensor0 = request.get_input_tensor(0) + input_tensor1 = request.get_input_tensor(1) + + if shared_flag: + if isinstance(input_data[0], np.ndarray) and input_data[0].dtype == input_tensor0.data.dtype: + assert np.shares_memory(input_data[0], input_tensor0.data) + else: + assert not np.shares_memory(input_data[0], input_tensor0.data) + if isinstance(input_data[1], np.ndarray) and input_data[1].dtype == input_tensor1.data.dtype: + assert np.shares_memory(input_data[1], input_tensor1.data) + else: + assert not np.shares_memory(input_data[1], input_tensor1.data) + else: + assert not np.shares_memory(input_data[0], input_tensor0.data) + assert not np.shares_memory(input_data[1], input_tensor1.data) + + +@pytest.mark.parametrize("shared_flag", [True, False]) +@pytest.mark.parametrize("input_data", [ + {0: np.array(1.0, dtype=np.float32), 1: np.array([3.0], dtype=np.float32)}, + {0: np.array(1.0, dtype=np.float32), 1: np.array([3.0, 3.0, 3.0], dtype=np.float32)}, +]) +def test_mixed_dynamic_infer(device, shared_flag, input_data): + core = Core() + param0 = ops.parameter([], np.float32, name="data0") + param1 = ops.parameter(["?"], np.float32, name="data1") + add = ops.add(param0, param1, name="add") + model = Model([add], [param0, param1], "mixed_model") + + compiled = core.compile_model(model=model, device_name=device) + request = compiled.create_infer_request() + + res = request.infer(input_data, shared_memory=shared_flag) + + assert np.allclose(res[request.model_outputs[0]], np.add(input_data[0], input_data[1])) + + input_tensor0 = request.get_input_tensor(0) + input_tensor1 = request.get_input_tensor(1) + + if shared_flag: + if isinstance(input_data[0], np.ndarray) and input_data[0].dtype == input_tensor0.data.dtype: + assert np.shares_memory(input_data[0], input_tensor0.data) + else: + assert not np.shares_memory(input_data[0], input_tensor0.data) + if isinstance(input_data[1], np.ndarray) and input_data[1].dtype == input_tensor1.data.dtype: + assert np.shares_memory(input_data[1], input_tensor1.data) + else: + assert not np.shares_memory(input_data[1], input_tensor1.data) + else: + assert not np.shares_memory(input_data[0], input_tensor0.data) + assert not np.shares_memory(input_data[1], input_tensor1.data) diff --git a/src/bindings/python/tests/test_runtime/test_memory_modes.py b/src/bindings/python/tests/test_runtime/test_memory_modes.py index ccbd44efa729bb..4f8864314ca4ee 100644 --- a/src/bindings/python/tests/test_runtime/test_memory_modes.py +++ b/src/bindings/python/tests/test_runtime/test_memory_modes.py @@ -125,3 +125,27 @@ def test_with_tensor_memory(cls, shared_flag_one, shared_flag_two, ov_type, nump else: assert not (np.shares_memory(arr, ov_object.data)) assert not (np.shares_memory(ov_tensor.data, ov_object.data)) + + +@pytest.mark.parametrize("cls", [Tensor, Constant]) +@pytest.mark.parametrize("shared_flag", [True, False]) +@pytest.mark.parametrize("scalar", [ + np.array(2), + np.array(1.0), + np.float32(3.0), + np.int64(7.0), + 4, + 5.0, +]) +def test_with_scalars(cls, shared_flag, scalar): + # If scalar is 0-dim np.array, create a copy for convinience. Otherwise, it will be + # shared by all tests. + # If scalar is np.number or native int/float, create 0-dim scalar array from it. + _scalar = np.copy(scalar) if isinstance(scalar, np.ndarray) else np.array(scalar) + ov_object = cls(array=_scalar, shared_memory=shared_flag) + if shared_flag is True: + assert np.shares_memory(_scalar, ov_object.data) + _scalar[()] = 6 + assert ov_object.data == 6 + else: + assert not (np.shares_memory(_scalar, ov_object.data)) diff --git a/src/bindings/python/tests/test_utils/test_data_dispatch.py b/src/bindings/python/tests/test_utils/test_data_dispatch.py index a0cbe5c8a73de1..254cf890458bb8 100644 --- a/src/bindings/python/tests/test_utils/test_data_dispatch.py +++ b/src/bindings/python/tests/test_utils/test_data_dispatch.py @@ -7,7 +7,7 @@ import numpy as np from tests.conftest import model_path -from tests.test_utils.test_utils import get_relu_model, generate_image, generate_model_and_image, generate_relu_compiled_model +from tests.test_utils.test_utils import generate_relu_compiled_model from openvino.runtime import Model, ConstOutput, Type, Shape, Core, Tensor from openvino.runtime.utils.data_helpers import _data_dispatch @@ -19,8 +19,8 @@ def _get_value(value): return value.data if isinstance(value, Tensor) else value -def _run_dispatcher(device, input_data, input_shape, is_shared): - compiled_model = generate_relu_compiled_model(device, input_shape) +def _run_dispatcher(device, input_data, is_shared, input_shape, input_dtype=np.float32): + compiled_model = generate_relu_compiled_model(device, input_shape, input_dtype) infer_request = compiled_model.create_infer_request() result = _data_dispatch(infer_request, input_data, is_shared) @@ -30,18 +30,58 @@ def _run_dispatcher(device, input_data, input_shape, is_shared): @pytest.mark.parametrize("data_type", [np.float_, np.int_, int, float]) @pytest.mark.parametrize("input_shape", [[], [1]]) @pytest.mark.parametrize("is_shared", [True, False]) -def test_scalars_dispatcher(device, data_type, input_shape, is_shared): +def test_scalars_dispatcher_old(device, data_type, input_shape, is_shared): test_data = data_type(2) expected = Tensor(np.ndarray([], data_type, np.array(test_data))) - result, _ = _run_dispatcher(device, test_data, input_shape, is_shared) + result, _ = _run_dispatcher(device, test_data, is_shared, input_shape) assert isinstance(result, Tensor) assert result.get_shape() == Shape([]) - assert result.get_element_type() == Type(data_type) + assert result.get_element_type() == Type(np.float32) assert result.data == expected.data +@pytest.mark.parametrize(("input_data", "input_dtype"), [ + (np.float_(2), np.float_), + (np.int_(1), np.int_), + (int(3), np.int64), + (1, np.int64), + (float(7), np.float64), + (1.0, np.int64), +]) +@pytest.mark.parametrize("input_shape", [[], [1]]) +@pytest.mark.parametrize("is_shared", [True, False]) +def test_scalars_dispatcher_new_0(device, input_data, input_dtype, input_shape, is_shared): + expected = Tensor(np.array(input_data, dtype=input_dtype)) + + result, _ = _run_dispatcher(device, input_data, is_shared, input_shape, input_dtype) + + assert isinstance(result, Tensor) + assert result.get_shape() == Shape([]) + assert result.get_element_type() == Type(input_dtype) + assert result.data == expected.data + + +@pytest.mark.parametrize(("input_data", "is_shared", "expected"), [ + (np.array(2.0, dtype=np.float32), False, {}), + (np.array(2.0, dtype=np.float32), True, Tensor(np.array(2.0, dtype=np.float32))), + (np.array(1, dtype=np.int8), False, {}), + (np.array(1, dtype=np.int8), True, Tensor(np.array(1, dtype=np.float32))), +]) +@pytest.mark.parametrize("input_shape", [[], [1]]) +def test_scalars_dispatcher_new_1(device, input_data, is_shared, expected, input_shape): + result, _ = _run_dispatcher(device, input_data, is_shared, input_shape, np.float32) + + assert isinstance(result, type(expected)) + if isinstance(result, dict): + assert len(result) == 0 + else: + assert result.get_shape() == Shape(input_shape) + assert result.get_element_type() == Type(np.float32) + assert result.data == expected.data + + @pytest.mark.parametrize("input_shape", [[1], [2, 2]]) @pytest.mark.parametrize("is_shared", [True, False]) def test_tensor_dispatcher(device, input_shape, is_shared): @@ -49,7 +89,7 @@ def test_tensor_dispatcher(device, input_shape, is_shared): test_data = Tensor(array, is_shared) - result, _ = _run_dispatcher(device, test_data, input_shape, is_shared) + result, _ = _run_dispatcher(device, test_data, is_shared, input_shape) assert isinstance(result, Tensor) assert result.get_shape() == Shape(input_shape) @@ -66,7 +106,7 @@ def test_tensor_dispatcher(device, input_shape, is_shared): def test_ndarray_shared_dispatcher(device, input_shape): test_data = np.ones(input_shape).astype(np.float32) - result, _ = _run_dispatcher(device, test_data, input_shape, True) + result, _ = _run_dispatcher(device, test_data, True, input_shape) assert isinstance(result, Tensor) assert result.get_shape() == Shape(test_data.shape) @@ -82,7 +122,7 @@ def test_ndarray_shared_dispatcher(device, input_shape): def test_ndarray_shared_dispatcher_casting(device, input_shape): test_data = np.ones(input_shape) - result, infer_request = _run_dispatcher(device, test_data, input_shape, True) + result, infer_request = _run_dispatcher(device, test_data, True, input_shape) assert isinstance(result, Tensor) assert result.get_shape() == Shape(test_data.shape) @@ -98,7 +138,7 @@ def test_ndarray_shared_dispatcher_casting(device, input_shape): def test_ndarray_shared_dispatcher_misalign(device, input_shape): test_data = np.asfortranarray(np.ones(input_shape).astype(np.float32)) - result, _ = _run_dispatcher(device, test_data, input_shape, True) + result, _ = _run_dispatcher(device, test_data, True, input_shape) assert isinstance(result, Tensor) assert result.get_shape() == Shape(test_data.shape) @@ -114,7 +154,7 @@ def test_ndarray_shared_dispatcher_misalign(device, input_shape): def test_ndarray_copied_dispatcher(device, input_shape): test_data = np.ones(input_shape) - result, infer_request = _run_dispatcher(device, test_data, input_shape, False) + result, infer_request = _run_dispatcher(device, test_data, False, input_shape) assert result == {} assert np.array_equal(infer_request.inputs[0].data, test_data) diff --git a/src/bindings/python/tests/test_utils/test_utils.py b/src/bindings/python/tests/test_utils/test_utils.py index 2aa3d68f038921..aa1f600e070099 100644 --- a/src/bindings/python/tests/test_utils/test_utils.py +++ b/src/bindings/python/tests/test_utils/test_utils.py @@ -47,10 +47,10 @@ def generate_image(shape: Tuple = (1, 3, 32, 32), dtype: Union[str, np.dtype] = return np.random.rand(*shape).astype(dtype) -def get_relu_model(input_shape: List[int] = None) -> openvino.runtime.Model: +def get_relu_model(input_shape: List[int] = None, input_dtype=np.float32) -> openvino.runtime.Model: if input_shape is None: input_shape = [1, 3, 32, 32] - param = ops.parameter(input_shape, np.float32, name="data") + param = ops.parameter(input_shape, input_dtype, name="data") relu = ops.relu(param, name="relu") model = Model([relu], [param], "test_model") model.get_ordered_ops()[2].friendly_name = "friendly" @@ -59,10 +59,14 @@ def get_relu_model(input_shape: List[int] = None) -> openvino.runtime.Model: return model -def generate_relu_compiled_model(device, input_shape: List[int] = None) -> openvino.runtime.CompiledModel: +def generate_relu_compiled_model( + device, + input_shape: List[int] = None, + input_dtype=np.float32, +) -> openvino.runtime.CompiledModel: if input_shape is None: input_shape = [1, 3, 32, 32] - model = get_relu_model(input_shape) + model = get_relu_model(input_shape, input_dtype) core = Core() return core.compile_model(model, device, {})