From 465d2c2b71478263ee15eaef7046e38cd3e5343f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 21 Feb 2024 16:22:28 +0100 Subject: [PATCH 1/2] Implemented dpnp.frombuffer, dpnp.fromfile and dpnp.fromstring --- dpnp/dpnp_iface_arraycreation.py | 275 +++++++++++++++++- tests/test_arraycreation.py | 17 +- tests/test_sycl_queue.py | 30 +- tests/test_usm_type.py | 28 +- .../cupy/creation_tests/test_from_data.py | 3 - 5 files changed, 322 insertions(+), 31 deletions(-) diff --git a/dpnp/dpnp_iface_arraycreation.py b/dpnp/dpnp_iface_arraycreation.py index 854c0c74313..27d6cfe7d96 100644 --- a/dpnp/dpnp_iface_arraycreation.py +++ b/dpnp/dpnp_iface_arraycreation.py @@ -1399,23 +1399,114 @@ def eye( ) -def frombuffer(buffer, **kwargs): - """ +def frombuffer( + buffer, + dtype=float, + count=-1, + offset=0, + *, + like=None, + device=None, + usm_type="device", + sycl_queue=None, +): + r""" Interpret a buffer as a 1-dimensional array. For full documentation refer to :obj:`numpy.frombuffer`. + Parameters + ---------- + buffer : buffer_like + An object that exposes the buffer interface. + dtype : data-type, optional + Data-type of the returned array. + Default is the default floating point data type for the device where + the returned array is allocated. + count : int, optional + Number of items to read. ``-1`` means all data in the buffer. + offset : int, optional + Start reading the buffer from this offset (in bytes); default: 0. + device : {None, string, SyclDevice, SyclQueue}, optional + An array API concept of device where the output array is created. + The `device` can be ``None`` (the default), an OneAPI filter selector + string, an instance of :class:`dpctl.SyclDevice` corresponding to + a non-partitioned SYCL device, an instance of :class:`dpctl.SyclQueue`, + or a `Device` object returned by + :obj:`dpnp.dpnp_array.dpnp_array.device` property. + usm_type : {"device", "shared", "host"}, optional + The type of SYCL USM allocation for the output array. + Default is "device". + sycl_queue : {None, SyclQueue}, optional + A SYCL queue to use for output array allocation and copying. + + Returns + ------- + out : dpnp.ndarray + A 1-dimensional array created from input buffer object. + Limitations ----------- - Only float64, float32, int64, int32 types are supported. + Parameter `like` is supported only with default value ``None``. + Otherwise, the function raises `NotImplementedError` exception. - """ + Notes + ----- + This uses :obj:`numpy.frombuffer` and coerces the result to a DPNP array. - return call_origin(numpy.frombuffer, buffer, **kwargs) + See also + -------- + :obj:`dpnp.fromfile` : Construct array from data in a text or binary file. + :obj:`dpnp.fromiter` : Construct array from an iterable object. + :obj:`dpnp.fromstring` : Construct array from the text data in a string. + Examples + -------- + >>> import dpnp as np + >>> s = b'\x01\x02\x03\x04' + >>> np.frombuffer(s, dtype=np.int32) + array([67305985], dtype=int32) + >>> np.frombuffer(b'\x01\x02\x03\x04\x05', dtype=numpy.uint8, count=3) + array([1, 2, 3], dtype=uint8) + + Creating an array on a different device or with a specified usm_type + + >>> x = np.frombuffer(s, dtype=np.int32) # default case + >>> x.device, x.usm_type + (Device(level_zero:gpu:0), 'device') + + >>> y = np.frombuffer(s, dtype=np.int32, device='cpu') + >>> y.device, y.usm_type + (Device(opencl:cpu:0), 'device') + + >>> z = np.frombuffer(s, dtype=np.int32, usm_type="host") + >>> z.device, z.usm_type + (Device(level_zero:gpu:0), 'host') -def fromfile(file, **kwargs): """ + + _check_limitations(like=like) + return asarray( + numpy.frombuffer(buffer, dtype=dtype, count=count, offset=offset), + device=device, + usm_type=usm_type, + sycl_queue=sycl_queue, + ) + + +def fromfile( + file, + dtype=float, + count=-1, + sep="", + offset=0, + *, + like=None, + device=None, + usm_type="device", + sycl_queue=None, +): + r""" Construct an array from data in a text or binary file. A highly efficient way of reading binary data with a known data-type, @@ -1424,13 +1515,107 @@ def fromfile(file, **kwargs): For full documentation refer to :obj:`numpy.fromfile`. + Parameters + ---------- + file : file or str or Path + Open file object or filename. + dtype : data-type, optional + Data type of the returned array. + For binary files, it is used to determine the size and byte-order + of the items in the file. + Default is the default floating point data type for the device where + the returned array is allocated. + count : int, optional + Number of items to read. ``-1`` means all items (i.e., the complete + file). + sep : str, optional + Separator between items if `file` is a text file. + Empty ("") separator means the file should be treated as binary. + Spaces (" ") in the separator match zero or more whitespace characters. + A separator consisting only of spaces must match at least one + whitespace. + offset : int, optional + The offset (in bytes) from the file's current position. Defaults to 0. + Only permitted for binary files. + device : {None, string, SyclDevice, SyclQueue}, optional + An array API concept of device where the output array is created. + The `device` can be ``None`` (the default), an OneAPI filter selector + string, an instance of :class:`dpctl.SyclDevice` corresponding to + a non-partitioned SYCL device, an instance of :class:`dpctl.SyclQueue`, + or a `Device` object returned by + :obj:`dpnp.dpnp_array.dpnp_array.device` property. + usm_type : {"device", "shared", "host"}, optional + The type of SYCL USM allocation for the output array. + Default is "device". + sycl_queue : {None, SyclQueue}, optional + A SYCL queue to use for output array allocation and copying. + + Returns + ------- + out : dpnp.ndarray + A 1-dimensional array created from data in a text or binary file. + Limitations ----------- - Only float64, float32, int64, int32 types are supported. + Parameter `like` is supported only with default value ``None``. + Otherwise, the function raises `NotImplementedError` exception. + + Notes + ----- + This uses :obj:`numpy.fromfile` and coerces the result to a DPNP array. + + See also + -------- + :obj:`dpnp.frombuffer` : Construct array from the buffer data. + :obj:`dpnp.fromiter` : Construct array from an iterable object. + :obj:`dpnp.fromstring` : Construct array from the text data in a string. + :obj:`dpnp.load` : Load arrays or pickled objects from files. + :obj:`dpnp.save` : Save an array to a binary file. + :obj:`dpnp.ndarray.tofile` : Write array to a file as text or binary. + :obj:`dpnp.loadtxt` : More flexible way of loading data from a text file. + + Examples + -------- + Save the data to a temporary file: + + >>> import tempfile + >>> x = tempfile.TemporaryFile() + >>> x.write(b"\x00\x01\x02\x03\x04") + >>> x.flush() + >>> x.seek(0) + + Construct an array: + + >>> import dpnp as np + >>> np.fromfile(x, dtype="u1") + array([0, 1, 2, 3, 4], dtype=uint8) + + Creating an array on a different device or with a specified usm_type + + >>> x.seek(0) + >>> x = np.fromfile(x, dtype="u1") # default case + >>> x.device, x.usm_type + (Device(level_zero:gpu:0), 'device') + + >>> x.seek(0) + >>> y = np.fromfile(x, dtype="u1", device='cpu') + >>> y.device, y.usm_type + (Device(opencl:cpu:0), 'device') + + >>> x.seek(0) + >>> z = np.fromfile(x, dtype="u1", usm_type="host") + >>> z.device, z.usm_type + (Device(level_zero:gpu:0), 'host') """ - return call_origin(numpy.fromfile, file, **kwargs) + _check_limitations(like=like) + return asarray( + numpy.fromfile(file, dtype=dtype, count=count, sep=sep, offset=offset), + device=device, + usm_type=usm_type, + sycl_queue=sycl_queue, + ) def fromfunction(function, shape, **kwargs): @@ -1466,19 +1651,87 @@ def fromiter(iterable, dtype, count=-1): return call_origin(numpy.fromiter, iterable, dtype, count) -def fromstring(string, **kwargs): +def fromstring( + string, + dtype=float, + count=-1, + *, + sep, + like=None, + device=None, + usm_type="device", + sycl_queue=None, +): """ A new 1-D array initialized from text data in a string. For full documentation refer to :obj:`numpy.fromstring`. + Parameters + ---------- + string : str + A string containing the data. + dtype : data-type, optional + The data type of the array. + For binary input data, the data must be in exactly this format. + Default is the default floating point data type for the device where + the returned array is allocated. + count : int, optional + Read this number of `dtype` elements from the data. If this is negative + (the default), the count will be determined from the length of the data. + sep : str, optional + The string separating numbers in the data; extra whitespace between + elements is also ignored. + device : {None, string, SyclDevice, SyclQueue}, optional + An array API concept of device where the output array is created. + The `device` can be ``None`` (the default), an OneAPI filter selector + string, an instance of :class:`dpctl.SyclDevice` corresponding to + a non-partitioned SYCL device, an instance of :class:`dpctl.SyclQueue`, + or a `Device` object returned by + :obj:`dpnp.dpnp_array.dpnp_array.device` property. + usm_type : {"device", "shared", "host"}, optional + The type of SYCL USM allocation for the output array. + Default is "device". + sycl_queue : {None, SyclQueue}, optional + A SYCL queue to use for output array allocation and copying. + + Returns + ------- + out : dpnp.ndarray + The constructed array. + Limitations ----------- - Only float64, float32, int64, int32 types are supported. + Parameter `like` is supported only with default value ``None``. + Otherwise, the function raises `NotImplementedError` exception. + + Notes + ----- + This uses :obj:`numpy.fromstring` and coerces the result to a DPNP array. + + See also + -------- + :obj:`dpnp.frombuffer` : Construct array from the buffer data. + :obj:`dpnp.fromfile` : Construct array from data in a text or binary file. + :obj:`dpnp.fromiter` : Construct array from an iterable object. + + Examples + -------- + >>> import dpnp as np + >>> np.fromstring('1 2', dtype=int, sep=' ') + array([1, 2]) + >>> np.fromstring('1, 2', dtype=int, sep=',') + array([1, 2]) """ - return call_origin(numpy.fromstring, string, **kwargs) + _check_limitations(like=like) + return asarray( + numpy.fromstring(string, dtype=dtype, count=count, sep=sep), + device=device, + usm_type=usm_type, + sycl_queue=sycl_queue, + ) def full( diff --git a/tests/test_arraycreation.py b/tests/test_arraycreation.py index 7e70af6b765..3b097a41d9c 100644 --- a/tests/test_arraycreation.py +++ b/tests/test_arraycreation.py @@ -52,6 +52,7 @@ def test_exception_order(func, args): pytest.param("asfortranarray", [2]), pytest.param("empty", [(2,)]), pytest.param("eye", [2]), + pytest.param("frombuffer", [b"\x01\x02\x03\x04"]), pytest.param("full", [(2,), 4]), pytest.param("identity", [2]), pytest.param("ones", [(2,)]), @@ -208,21 +209,14 @@ def test_eye(N, M, k, dtype, order): assert_array_equal(func(numpy), func(dpnp)) -@pytest.mark.usefixtures("allow_fall_back_on_numpy") -@pytest.mark.parametrize( - "dtype", - get_all_dtypes( - no_float16=False, no_none=False if has_support_aspect64() else True - ), -) +@pytest.mark.parametrize("dtype", get_all_dtypes(no_float16=False)) def test_frombuffer(dtype): buffer = b"12345678ABCDEF00" func = lambda xp: xp.frombuffer(buffer, dtype=dtype) - assert_allclose(func(dpnp), func(numpy)) + assert_dtype_allclose(func(dpnp), func(numpy)) -@pytest.mark.usefixtures("allow_fall_back_on_numpy") -@pytest.mark.parametrize("dtype", get_all_dtypes()) +@pytest.mark.parametrize("dtype", get_all_dtypes(no_float16=False)) def test_fromfile(dtype): with tempfile.TemporaryFile() as fh: fh.write(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08") @@ -236,7 +230,7 @@ def test_fromfile(dtype): fh.seek(0) dpnp_res = func(dpnp) - assert_almost_equal(dpnp_res, np_res) + assert_dtype_allclose(dpnp_res, np_res) @pytest.mark.usefixtures("allow_fall_back_on_numpy") @@ -260,7 +254,6 @@ def test_fromiter(dtype): assert_array_equal(func(dpnp), func(numpy)) -@pytest.mark.usefixtures("allow_fall_back_on_numpy") @pytest.mark.parametrize("dtype", get_all_dtypes(no_float16=False)) def test_fromstring(dtype): string = "1 2 3 4" diff --git a/tests/test_sycl_queue.py b/tests/test_sycl_queue.py index dd6a9f5c639..ed8eeb2bdea 100644 --- a/tests/test_sycl_queue.py +++ b/tests/test_sycl_queue.py @@ -1,3 +1,5 @@ +import tempfile + import dpctl import dpctl.tensor as dpt import numpy @@ -83,6 +85,10 @@ def vvsort(val, vec, size, xp): "func, arg, kwargs", [ pytest.param("arange", [-25.7], {"stop": 10**8, "step": 15}), + pytest.param( + "frombuffer", [b"\x01\x02\x03\x04"], {"dtype": dpnp.int32} + ), + pytest.param("fromstring", ["1, 2"], {"dtype": int, "sep": " "}), pytest.param("full", [(2, 2)], {"fill_value": 5}), pytest.param("eye", [4, 2], {}), pytest.param("geomspace", [1, 4, 8], {}), @@ -104,7 +110,9 @@ def test_array_creation(func, arg, kwargs, device): dpnp_kwargs["device"] = device dpnp_array = getattr(dpnp, func)(*arg, **dpnp_kwargs) - numpy_array = getattr(numpy, func)(*arg, dtype=dpnp_array.dtype, **kwargs) + numpy_kwargs = dict(kwargs) + numpy_kwargs["dtype"] = dpnp_array.dtype + numpy_array = getattr(numpy, func)(*arg, **numpy_kwargs) assert_dtype_allclose(dpnp_array, numpy_array) assert dpnp_array.sycl_device == device @@ -311,6 +319,26 @@ def test_array_creation_cross_device_2d_array( assert_sycl_queue_equal(y.sycl_queue, x.to_device(device_y).sycl_queue) +@pytest.mark.parametrize( + "device", + valid_devices, + ids=[device.filter_string for device in valid_devices], +) +def test_array_creation_from_file(device): + with tempfile.TemporaryFile() as fh: + fh.write(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08") + fh.flush() + + fh.seek(0) + numpy_array = numpy.fromfile(fh) + + fh.seek(0) + dpnp_array = dpnp.fromfile(fh, device=device) + + assert_dtype_allclose(dpnp_array, numpy_array) + assert dpnp_array.sycl_device == device + + @pytest.mark.parametrize( "device_x", valid_devices, diff --git a/tests/test_usm_type.py b/tests/test_usm_type.py index 26c7e9e3057..d66ff59dc7d 100644 --- a/tests/test_usm_type.py +++ b/tests/test_usm_type.py @@ -1,10 +1,10 @@ +import tempfile from math import prod import dpctl.tensor as dpt import dpctl.utils as du import numpy import pytest -from numpy.testing import assert_allclose import dpnp as dp @@ -193,6 +193,8 @@ def test_array_creation_from_2d_array(func, args, usm_type_x, usm_type_y): "func, arg, kwargs", [ pytest.param("arange", [-25.7], {"stop": 10**8, "step": 15}), + pytest.param("frombuffer", [b"\x01\x02\x03\x04"], {"dtype": dp.int32}), + pytest.param("fromstring", ["1, 2"], {"dtype": int, "sep": " "}), pytest.param("full", [(2, 2)], {"fill_value": 5}), pytest.param("eye", [4, 2], {}), pytest.param("geomspace", [1, 4, 8], {}), @@ -209,12 +211,30 @@ def test_array_creation_from_scratch(func, arg, kwargs, usm_type): dpnp_kwargs = dict(kwargs) dpnp_kwargs["usm_type"] = usm_type dpnp_array = getattr(dp, func)(*arg, **dpnp_kwargs) - numpy_array = getattr(numpy, func)(*arg, dtype=dpnp_array.dtype, **kwargs) - tol = 1e-06 - assert_allclose(dpnp_array, numpy_array, rtol=tol, atol=tol) + numpy_kwargs = dict(kwargs) + numpy_kwargs["dtype"] = dpnp_array.dtype + numpy_array = getattr(numpy, func)(*arg, **numpy_kwargs) + + assert_dtype_allclose(dpnp_array, numpy_array) assert dpnp_array.shape == numpy_array.shape + assert dpnp_array.usm_type == usm_type + + +@pytest.mark.parametrize("usm_type", list_of_usm_types, ids=list_of_usm_types) +def test_array_creation_from_file(usm_type): + with tempfile.TemporaryFile() as fh: + fh.write(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08") + fh.flush() + + fh.seek(0) + numpy_array = numpy.fromfile(fh) + + fh.seek(0) + dpnp_array = dp.fromfile(fh, usm_type=usm_type) + assert_dtype_allclose(dpnp_array, numpy_array) + assert dpnp_array.shape == numpy_array.shape assert dpnp_array.usm_type == usm_type diff --git a/tests/third_party/cupy/creation_tests/test_from_data.py b/tests/third_party/cupy/creation_tests/test_from_data.py index a517a2cb2a1..0f24d201688 100644 --- a/tests/third_party/cupy/creation_tests/test_from_data.py +++ b/tests/third_party/cupy/creation_tests/test_from_data.py @@ -543,7 +543,6 @@ def test_asfortranarray_cuda_array_zero_dim_dtype( a = xp.ones((), dtype=dtype_a) return xp.asfortranarray(a, dtype=dtype_b) - @pytest.mark.usefixtures("allow_fall_back_on_numpy") @testing.numpy_cupy_array_equal() def test_fromfile(self, xp): with tempfile.TemporaryFile() as fh: @@ -566,12 +565,10 @@ def test_fromiter(self, xp): iterable = (x * x for x in range(5)) return xp.fromiter(iterable, float) - @pytest.mark.usefixtures("allow_fall_back_on_numpy") @testing.numpy_cupy_array_equal() def test_fromstring(self, xp): return xp.fromstring("1 2", dtype=int, sep=" ") - @pytest.mark.usefixtures("allow_fall_back_on_numpy") @testing.numpy_cupy_array_equal() def test_frombuffer(self, xp): return xp.frombuffer(b"\x01\x02", dtype=numpy.uint8) From e6dc2ea305e40475d9de1892b6c93b66ce1c2888 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Feb 2024 13:25:00 +0100 Subject: [PATCH 2/2] Address the comments --- dpnp/dpnp_iface_arraycreation.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dpnp/dpnp_iface_arraycreation.py b/dpnp/dpnp_iface_arraycreation.py index 27d6cfe7d96..c3999594fe3 100644 --- a/dpnp/dpnp_iface_arraycreation.py +++ b/dpnp/dpnp_iface_arraycreation.py @@ -1466,7 +1466,7 @@ def frombuffer( >>> s = b'\x01\x02\x03\x04' >>> np.frombuffer(s, dtype=np.int32) array([67305985], dtype=int32) - >>> np.frombuffer(b'\x01\x02\x03\x04\x05', dtype=numpy.uint8, count=3) + >>> np.frombuffer(b'\x01\x02\x03\x04\x05', dtype='u1', count=3) array([1, 2, 3], dtype=uint8) Creating an array on a different device or with a specified usm_type @@ -1579,31 +1579,31 @@ def fromfile( Save the data to a temporary file: >>> import tempfile - >>> x = tempfile.TemporaryFile() - >>> x.write(b"\x00\x01\x02\x03\x04") - >>> x.flush() - >>> x.seek(0) + >>> fh = tempfile.TemporaryFile() + >>> fh.write(b"\x00\x01\x02\x03\x04") + >>> fh.flush() + >>> fh.seek(0) Construct an array: >>> import dpnp as np - >>> np.fromfile(x, dtype="u1") + >>> np.fromfile(fh, dtype="u1") array([0, 1, 2, 3, 4], dtype=uint8) Creating an array on a different device or with a specified usm_type - >>> x.seek(0) - >>> x = np.fromfile(x, dtype="u1") # default case + >>> fh.seek(0) + >>> x = np.fromfile(fh, dtype="u1") # default case >>> x.device, x.usm_type (Device(level_zero:gpu:0), 'device') - >>> x.seek(0) - >>> y = np.fromfile(x, dtype="u1", device='cpu') + >>> fh.seek(0) + >>> y = np.fromfile(fh, dtype="u1", device='cpu') >>> y.device, y.usm_type (Device(opencl:cpu:0), 'device') - >>> x.seek(0) - >>> z = np.fromfile(x, dtype="u1", usm_type="host") + >>> fh.seek(0) + >>> z = np.fromfile(fh, dtype="u1", usm_type="host") >>> z.device, z.usm_type (Device(level_zero:gpu:0), 'host')