From c6b1bcb888523089fc91e872a8c91addb2db6fac Mon Sep 17 00:00:00 2001 From: eric-haibin-lin Date: Sun, 30 Jul 2017 04:14:44 +0000 Subject: [PATCH 1/2] support mx.nd.empty for sparse ndarray Change SparseNDArray to BaseSparseNDArray support mx.nd.array with BaseSparseNDArray inputs. Update documentation with explicit subclasses of NDArrays Conflicts: python/mxnet/ndarray/__init__.py python/mxnet/ndarray/ndarray.py python/mxnet/ndarray/sparse_ndarray.py tests/python/unittest/test_sparse_ndarray.py --- python/mxnet/ndarray/__init__.py | 10 +- python/mxnet/ndarray/ndarray.py | 85 ++++----- python/mxnet/ndarray/ndarray_utils.py | 100 +++++++++- python/mxnet/ndarray/sparse_ndarray.py | 181 +++++++++++++------ python/mxnet/test_utils.py | 3 +- tests/python/unittest/test_sparse_ndarray.py | 22 ++- 6 files changed, 271 insertions(+), 130 deletions(-) diff --git a/python/mxnet/ndarray/__init__.py b/python/mxnet/ndarray/__init__.py index f9e935b76d25..f8d77dcdeaa9 100644 --- a/python/mxnet/ndarray/__init__.py +++ b/python/mxnet/ndarray/__init__.py @@ -3,10 +3,10 @@ from . import _internal from . import op from .op import CachedOp -from .ndarray import NDArray, array, concatenate, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP -from .ndarray import empty, ones, add, arange, divide, equal, full, greater, greater_equal, imdecode +from .ndarray import NDArray, concatenate, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP +from .ndarray import ones, add, arange, divide, equal, full, greater, greater_equal, imdecode from .ndarray import lesser, lesser_equal, maximum, minimum, moveaxis, multiply, negative, not_equal from .ndarray import onehot_encode, power, subtract, true_divide, waitall, _new_empty_handle -from .ndarray_utils import load, save, zeros -from .sparse_ndarray import _ndarray_cls -from .sparse_ndarray import csr, row_sparse, SparseNDArray, todense, RowSparseNDArray, CSRNDArray +from .ndarray_utils import load, save, zeros, empty, array +from .sparse_ndarray import _ndarray_cls, todense +from .sparse_ndarray import csr, row_sparse, BaseSparseNDArray, RowSparseNDArray, CSRNDArray diff --git a/python/mxnet/ndarray/ndarray.py b/python/mxnet/ndarray/ndarray.py index c45cefaedbc0..a027321d3597 100644 --- a/python/mxnet/ndarray/ndarray.py +++ b/python/mxnet/ndarray/ndarray.py @@ -921,7 +921,7 @@ def astype(self, dtype): >>> y.dtype """ - res = empty(self.shape, ctx=self.context, dtype=dtype) + res = _empty_ndarray(self.shape, ctx=self.context, dtype=dtype) self.copyto(res) return res @@ -942,7 +942,7 @@ def copyto(self, other): Returns ------- - NDArray + NDArray, CSRNDArray, RowSparseNDArray The copied array. If ``other`` is an ``NDArray``, then the return value and ``other`` will point to the same ``NDArray``. @@ -1090,39 +1090,6 @@ def onehot_encode(indices, out): # pylint: enable= no-member, protected-access -def empty(shape, ctx=None, dtype=mx_real_t): - """Returns a new array of given shape and type, without initializing entries. - - Parameters - ---------- - shape : int or tuple of int - The shape of the empty array. - ctx : Context, optional - An optional device context (default is the current default context). - dtype : str or numpy.dtype, optional - An optional value type (default is `float32`). - - Returns - ------- - NDArray - A created array. - - Examples - -------- - >>> mx.nd.empty(1) - - >>> mx.nd.empty((1,2), mx.gpu(0)) - - >>> mx.nd.empty((1,2), mx.gpu(0), 'float16') - - """ - if isinstance(shape, integer_types): - shape = (shape, ) - if ctx is None: - ctx = Context.default_ctx - return NDArray(handle=_new_alloc_handle(shape, ctx, False, dtype)) - - def ones(shape, ctx=None, dtype=None, **kwargs): """Returns a new array filled with all ones, with the given shape and type. @@ -1190,12 +1157,11 @@ def full(shape, val, ctx=None, dtype=mx_real_t, out=None): >>> mx.nd.full((1, 2), 2.0, dtype='float16').asnumpy() array([[ 2., 2.]], dtype=float16) """ - out = empty(shape, ctx, dtype) if out is None else out + out = _empty_ndarray(shape, ctx, dtype) if out is None else out out[:] = val return out - -def array(source_array, ctx=None, dtype=None): +def _array(source_array, ctx=None, dtype=None): """Creates an array from any object exposing the array interface. Parameters @@ -1213,18 +1179,6 @@ def array(source_array, ctx=None, dtype=None): ------- NDArray An `NDArray` with the same contents as the `source_array`. - - Examples - -------- - >>> import numpy as np - >>> mx.nd.array([1, 2, 3]) - - >>> mx.nd.array([[1, 2], [3, 4]]) - - >>> mx.nd.array(np.zeros((3, 2))) - - >>> mx.nd.array(np.zeros((3, 2)), mx.gpu(0)) - """ if isinstance(source_array, NDArray): dtype = source_array.dtype if dtype is None else dtype @@ -1235,11 +1189,10 @@ def array(source_array, ctx=None, dtype=None): source_array = np.array(source_array, dtype=dtype) except: raise TypeError('source_array must be array like object') - arr = empty(source_array.shape, ctx, dtype) + arr = _empty_ndarray(source_array.shape, ctx, dtype) arr[:] = source_array return arr - def moveaxis(tensor, source, destination): """Moves the `source` axis into the `destination` position while leaving the other axes in their original order @@ -2290,7 +2243,7 @@ def concatenate(arrays, axis=0, always_copy=True): assert shape_rest2 == arr.shape[axis+1:] assert dtype == arr.dtype ret_shape = shape_rest1 + (shape_axis,) + shape_rest2 - ret = empty(ret_shape, ctx=arrays[0].context, dtype=dtype) + ret = _empty_ndarray(ret_shape, ctx=arrays[0].context, dtype=dtype) idx = 0 begin = [0 for _ in ret_shape] @@ -2387,3 +2340,29 @@ def _zeros_ndarray(shape, ctx=None, dtype=None, **kwargs): # pylint: disable= no-member, protected-access return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, **kwargs) # pylint: enable= no-member, protected-access + +def _empty_ndarray(shape, ctx=None, dtype=None): + """Returns a new array of given shape and type, without initializing entries. + + Parameters + ---------- + shape : int or tuple of int + The shape of the empty array. + ctx : Context, optional + An optional device context (default is the current default context). + dtype : str or numpy.dtype, optional + An optional value type (default is `float32`). + + Returns + ------- + NDArray + A created array. + + """ + if isinstance(shape, int): + shape = (shape, ) + if ctx is None: + ctx = Context.default_ctx + if dtype is None: + dtype = mx_real_t + return NDArray(handle=_new_alloc_handle(shape, ctx, False, dtype)) diff --git a/python/mxnet/ndarray/ndarray_utils.py b/python/mxnet/ndarray/ndarray_utils.py index 7531464cdf20..9f84c280b0d8 100644 --- a/python/mxnet/ndarray/ndarray_utils.py +++ b/python/mxnet/ndarray/ndarray_utils.py @@ -1,10 +1,11 @@ # coding: utf-8 -"""Utility functions for NDArray and SparseNDArray.""" +"""Utility functions for NDArray and BaseSparseNDArray.""" import ctypes from ..base import _LIB, check_call, py_str, c_str, string_types, mx_uint, NDArrayHandle, c_array -from .ndarray import NDArray, _zeros_ndarray -from .sparse_ndarray import _ndarray_cls, _zeros_sparse_ndarray +from .ndarray import NDArray, _zeros_ndarray, _empty_ndarray, _array +from .sparse_ndarray import _zeros_sparse_ndarray, _empty_sparse_ndarray, _sparse_array +from .sparse_ndarray import _ndarray_cls def zeros(shape, ctx=None, dtype=None, stype=None, aux_types=None, **kwargs): @@ -21,12 +22,12 @@ def zeros(shape, ctx=None, dtype=None, stype=None, aux_types=None, **kwargs): stype: string, optional The storage type of the empty array, such as 'row_sparse', 'csr', etc aux_types: list of numpy.dtype, optional - An optional type for the aux data for SparseNDArray (default values depends - on the storage type) + An optional type for the aux data for the BaseSparseNDArray (default values + depends on the storage type) Returns ------- - SparseNDArray + NDArray, CSRNDArray or RowSparseNDArray A created array Examples -------- @@ -36,11 +37,90 @@ def zeros(shape, ctx=None, dtype=None, stype=None, aux_types=None, **kwargs): array([[ 0., 0.]], dtype=float16) """ - if stype is None: + if stype is None or stype == 'default': return _zeros_ndarray(shape, ctx, dtype, **kwargs) else: return _zeros_sparse_ndarray(stype, shape, ctx, dtype, aux_types, **kwargs) +def empty(shape, ctx=None, dtype=None, stype=None, aux_types=None): + """Returns a new array of given shape and type, without initializing entries. + + Parameters + ---------- + shape : int or tuple of int + The shape of the empty array. + ctx : Context, optional + An optional device context (default is the current default context). + dtype : str or numpy.dtype, optional + An optional value type (default is `float32`). + stype : str, optional + An optional storage type (default is `default`). + aux_types: list of numpy.dtype, optional + An optional type for the aux data for the BaseSparseNDArray (default values depends + on the storage type) + + Returns + ------- + NDArray, CSRNDArray or RowSparseNDArray + A created array. + + Examples + -------- + >>> mx.nd.empty(1) + + >>> mx.nd.empty((1,2), mx.gpu(0)) + + >>> mx.nd.empty((1,2), mx.gpu(0), 'float16') + + >>> mx.nd.empty((1,2), stype='csr') + + """ + if stype is None or stype == 'default': + return _empty_ndarray(shape, ctx, dtype) + else: + return _empty_sparse_ndarray(stype, shape, ctx, dtype, aux_types) + +def array(source_array, ctx=None, dtype=None, aux_types=None): + """Creates an array from any object exposing the array interface. + + Parameters + ---------- + source_array : array_like + An object exposing the array interface, an object whose `__array__` + method returns an array, or any (nested) sequence. + ctx : Context, optional + Device context (default is the current default context). + dtype : str or numpy.dtype, optional + The data type of the output array. The default dtype is ``source_array.dtype`` + if `source_array` is an `NDArray`, `float32` otherwise. + aux_types: list of numpy.dtype, optional + An optional type for the aux data for the BaseSparseNDArray (default values + depends on the storage type) + + Returns + ------- + NDArray, RowSparseNDArray or CSRNDArray + An array with the same contents as the `source_array`. + + Examples + -------- + >>> import numpy as np + >>> mx.nd.array([1, 2, 3]) + + >>> mx.nd.array([[1, 2], [3, 4]]) + + >>> mx.nd.array(np.zeros((3, 2))) + + >>> mx.nd.array(np.zeros((3, 2)), mx.gpu(0)) + + >>> mx.nd.array(mx.nd.zeros((3, 2), stype='row_sparse')) + + """ + # TODO(haibin/anisub) Check if input is scipy.sparse object with `scipy.sparse.issparse` + if isinstance(source_array, NDArray) and source_array.stype != 'default': + return _sparse_array(source_array, ctx=ctx, dtype=dtype, aux_types=aux_types) + else: + return _array(source_array, ctx=ctx, dtype=dtype) def load(fname): """Loads an array from file. @@ -54,7 +134,8 @@ def load(fname): Returns ------- - list of NDArray or dict of str to NDArray + list of NDArray, RowSparseNDArray or CSRNDArray, or \ + dict of str to NDArray, RowSparseNDArray or CSRNDArray Loaded data. """ if not isinstance(fname, string_types): @@ -90,7 +171,8 @@ def save(fname, data): ---------- fname : str The filename. - data : list of ``NDArray` or dict of str to ``NDArray`` + data : list of NDArray, RowSparseNDArray or CSRNDArray, \ + or dict of str to NDArray, RowSparseNDArray or CSRNDArray The data to save. Examples diff --git a/python/mxnet/ndarray/sparse_ndarray.py b/python/mxnet/ndarray/sparse_ndarray.py index 04e826710d91..195f56283c25 100644 --- a/python/mxnet/ndarray/sparse_ndarray.py +++ b/python/mxnet/ndarray/sparse_ndarray.py @@ -21,10 +21,9 @@ from ..base import mx_uint, NDArrayHandle, check_call from ..context import Context from . import _internal -from . import ndarray from .ndarray import _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP from .ndarray import _STORAGE_TYPE_STR_TO_ID -from .ndarray import NDArray, _storage_type, _zeros_ndarray, array +from .ndarray import NDArray, _storage_type, _zeros_ndarray, _array from . import cast_storage from . import slice as nd_slice @@ -81,31 +80,26 @@ def _new_alloc_handle(stype, shape, ctx, delay_alloc, dtype, aux_types, aux_shap ctypes.byref(hdl))) return hdl +class BaseSparseNDArray(NDArray): + """The base class of an NDArray stored in a sparse storage format. -class SparseNDArray(NDArray): - """An array object representing a multidimensional, homogeneous array of - fixed-size items, stored in sparse format. See CSRNDArray and RowSparseNDArray - for more details. + See CSRNDArray and RowSparseNDArray for more details. """ + def __iadd__(self, other): - (self + other).copyto(self) - return self + raise NotImplementedError() def __isub__(self, other): - (self - other).copyto(self) - return self + raise NotImplementedError() def __imul__(self, other): - (self * other).copyto(self) - return self + raise NotImplementedError() def __idiv__(self, other): - (self / other).copyto(self) - return self + raise NotImplementedError() def __itruediv__(self, other): - (self / other).copyto(self) - return self + raise NotImplementedError() def __setitem__(self, key, value): """x.__setitem__(i, y) <=> x[i]=y @@ -126,14 +120,14 @@ def __setitem__(self, key, value): array([[ 1., 0., 2.], [ 0., 0., 0.], [ 4., 5., 6.]], dtype=float32) - >>> # assign SparseNDArray with same storage type + >>> # assign BaseSparseNDArray with same storage type >>> x = mx.nd.zeros('row_sparse', (3,3)) >>> x[:] = src >>> x.asnumpy() array([[ 1., 0., 2.], [ 0., 0., 0.], [ 4., 5., 6.]], dtype=float32) - >>> # assign NDArray to SparseNDArray + >>> # assign NDArray to BaseSparseNDArray >>> x[:] = mx.nd.ones((3,3)) >>> x.asnumpy() array([[ 1., 1., 1.], @@ -144,24 +138,24 @@ def __setitem__(self, key, value): raise ValueError('Failed to assign to a readonly NDArray') if isinstance(key, py_slice): if key.step is not None or key.start is not None or key.stop is not None: - raise ValueError('Assignment with slicing not supported in SparseNDArray.') + raise ValueError('Assignment with slicing not supported in BaseSparseNDArray.') if isinstance(value, NDArray): # avoid copying to itself if value.handle is not self.handle: value.copyto(self) elif isinstance(value, numeric_types): - raise Exception("Assigning numeric types to SparseNDArray not supported yet.") + raise Exception("Assigning numeric types to BaseSparseNDArray not supported yet.") elif isinstance(value, (np.ndarray, np.generic)): # TODO(haibin) Implement _sync_copyfrom for sparse ndarray to avoid an extra copy - warnings.warn('Assigning non-NDArray object to SparseNDArray is not efficient', + warnings.warn('Assigning non-NDArray object to BaseSparseNDArray is not efficient', RuntimeWarning) - tmp = ndarray.array(value) + tmp = _array(value) tmp.copyto(self) else: raise TypeError('type %s not supported' % str(type(value))) else: assert(isinstance(key, (int, tuple))) - raise Exception('SparseNDArray only supports [:] for assignment') + raise Exception('BaseSparseNDArray only supports [:] for assignment') def __getitem__(self, key): """x.__getitem__(i) <=> x[i] @@ -217,7 +211,7 @@ def _aux_type(self, i): Returns ------- numpy.dtype - This SparseNDArray's aux data type. + This BaseSparseNDArray's aux data type. """ aux_type = ctypes.c_int() check_call(_LIB.MXNDArrayGetAuxType(self.handle, i, ctypes.byref(aux_type))) @@ -225,14 +219,22 @@ def _aux_type(self, i): @property def data(self): - """Get a deep copy of the values array of the SparseNDArray. + """Get a deep copy NDArray of the data array associated with the BaseSparseNDArray. - Returns - ------- - NDArray - A deep copy of the SparseNDArray's values array. + This function blocks. Do not use it in performance critical code. + """ + self.wait_to_read() + hdl = NDArrayHandle() + check_call(_LIB.MXNDArrayGetDataNDArray(self.handle, ctypes.byref(hdl))) + return NDArray(hdl) + + @property + def indices(self): + """Get a deep copy NDArray of the indices array associated with the BaseSparseNDArray. + + This function blocks. Do not use it in performance critical code. """ - return self._data() + raise NotImplementedError() @property def _num_aux(self): @@ -242,7 +244,7 @@ def _num_aux(self): @property def _aux_types(self): - """The data types of the aux data for the SparseNDArray. + """The data types of the aux data for the BaseSparseNDArray. """ aux_types = [] num_aux = self._num_aux @@ -310,7 +312,9 @@ def todense(self): return todense(self) def _aux_data(self, i): - """ Get a deep copy NDArray of the i-th aux data array associated with the SparseNDArray. + """ Get a deep copy NDArray of the i-th aux data array associated with the + BaseSparseNDArray. + This function blocks. Do not use it in performance critical code. """ self.wait_to_read() @@ -318,17 +322,9 @@ def _aux_data(self, i): check_call(_LIB.MXNDArrayGetAuxNDArray(self.handle, i, ctypes.byref(hdl))) return NDArray(hdl) - def _data(self): - """ Get a deep copy NDArray of the value array associated with the SparseNDArray. - This function blocks. Do not use it in performance critical code. - """ - self.wait_to_read() - hdl = NDArrayHandle() - check_call(_LIB.MXNDArrayGetDataNDArray(self.handle, ctypes.byref(hdl))) - return NDArray(hdl) # pylint: disable=abstract-method -class CSRNDArray(SparseNDArray): +class CSRNDArray(BaseSparseNDArray): """A CSRNDArray represents a NDArray as three separate arrays: `data`, `indptr` and `indices`. It uses the standard CSR representation where the column indices for row i are stored in indices[indptr[i]:indptr[i+1]] and their corresponding values are stored @@ -349,32 +345,52 @@ class CSRNDArray(SparseNDArray): def __reduce__(self): return CSRNDArray, (None,), super(CSRNDArray, self).__getstate__() + def __iadd__(self, other): + (self + other).copyto(self) + return self + + def __isub__(self, other): + (self - other).copyto(self) + return self + + def __imul__(self, other): + (self * other).copyto(self) + return self + + def __idiv__(self, other): + (self / other).copyto(self) + return self + + def __itruediv__(self, other): + (self / other).copyto(self) + return self + @property def indices(self): - """The indices array of the SparseNDArray with `csr` storage type. + """The indices array of the CSRNDArray. This generates a deep copy of the column indices of the current `csr` matrix. Returns ------- NDArray - This SparseNDArray's indices array. + This CSRNDArray's indices array. """ return self._aux_data(1) @property def indptr(self): - """The indptr array of the SparseNDArray with `csr` storage type. + """The indptr array of the CSRNDArray with `csr` storage type. This generates a deep copy of the `indptr` of the current `csr` matrix. Returns ------- NDArray - This SparseNDArray's indptr array. + This CSRNDArray's indptr array. """ return self._aux_data(0) # pylint: disable=abstract-method -class RowSparseNDArray(SparseNDArray): +class RowSparseNDArray(BaseSparseNDArray): """A RowSparseNDArray is typically used to represent a subset of a larger NDArray with `default` of shape [LARGE0, D1, .. , DN] where LARGE0 >> D0. The values in indices are the indices in the first dimension of the slices that have been extracted from @@ -402,15 +418,35 @@ class RowSparseNDArray(SparseNDArray): def __reduce__(self): return RowSparseNDArray, (None,), super(RowSparseNDArray, self).__getstate__() + def __iadd__(self, other): + (self + other).copyto(self) + return self + + def __isub__(self, other): + (self - other).copyto(self) + return self + + def __imul__(self, other): + (self * other).copyto(self) + return self + + def __idiv__(self, other): + (self / other).copyto(self) + return self + + def __itruediv__(self, other): + (self / other).copyto(self) + return self + @property def indices(self): - """The indices array of the SparseNDArray with `row_sparse` storage type. + """The indices array of the RowSparseNDArray with `row_sparse` storage type. This generates a deep copy of the row indices of the current row-sparse matrix. Returns ------- NDArray - This SparseNDArray's indices array. + This RowSparseNDArray's indices array. """ return self._aux_data(0) @@ -493,11 +529,11 @@ def csr(data, indptr, indices, shape, ctx=None, dtype=None, indptr_type=None, in # if they are not for now. In the future, we should provide a c-api # to accept np.ndarray types to copy from to result.data and aux_data if not isinstance(data, NDArray): - data = array(data, ctx, dtype) + data = _array(data, ctx, dtype) if not isinstance(indptr, NDArray): - indptr = array(indptr, ctx, indptr_type) + indptr = _array(indptr, ctx, indptr_type) if not isinstance(indices, NDArray): - indices = array(indices, ctx, indices_type) + indices = _array(indices, ctx, indices_type) check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, data.handle, ctypes.c_int(-1))) check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indptr.handle, ctypes.c_int(0))) check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indices.handle, ctypes.c_int(1))) @@ -559,21 +595,21 @@ def row_sparse(data, indices, shape, ctx=None, dtype=None, indices_type=None): # if they are not for now. In the future, we should provide a c-api # to accept np.ndarray types to copy from to result.data and aux_data if not isinstance(data, NDArray): - data = array(data, ctx, dtype) + data = _array(data, ctx, dtype) if not isinstance(indices, NDArray): - indices = array(indices, ctx, indices_type) + indices = _array(indices, ctx, indices_type) check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, data.handle, ctypes.c_int(-1))) check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indices.handle, ctypes.c_int(0))) return result def todense(source): - """ Return a dense array representation of this SparseNDArray. + """ Return a dense array representation of a BaseSparseNDArray. Returns ------- NDArray - The dense array with default storage + A copy of the array with `default` storage stype """ return cast_storage(source, stype='default') @@ -608,12 +644,12 @@ def _zeros_sparse_ndarray(stype, shape, ctx=None, dtype=None, aux_types=None, ** dtype : str or numpy.dtype, optional An optional value type (default is `float32`) aux_types: list of numpy.dtype, optional - An optional type for the aux data for SparseNDArray (default values depends + An optional type for the aux data for BaseSparseNDArray (default values depends on the storage type) Returns ------- - SparseNDArray + RowSparseNDArray or CSRNDArray A created array Examples -------- @@ -635,3 +671,34 @@ def _zeros_sparse_ndarray(stype, shape, ctx=None, dtype=None, aux_types=None, ** assert(len(aux_types) == len(_STORAGE_AUX_TYPES[stype])) out = _ndarray_cls(_new_alloc_handle(stype, shape, ctx, True, dtype, aux_types)) return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, out=out, **kwargs) + +def _empty_sparse_ndarray(stype, shape, ctx=None, dtype=None, aux_types=None): + """Returns a new array of given shape and type, without initializing entries. + """ + if isinstance(shape, int): + shape = (shape, ) + if ctx is None: + ctx = Context.default_ctx + if dtype is None: + dtype = mx_real_t + assert(stype is not None) + if stype == 'csr' or stype == 'row_sparse': + return _zeros_sparse_ndarray(stype, shape, ctx=ctx, dtype=dtype, aux_types=aux_types) + else: + raise Exception("unknown stype : " + str(stype)) + +def _sparse_array(source_array, ctx=None, dtype=None, aux_types=None): + """Creates a sparse array from any object exposing the array interface. + """ + if isinstance(source_array, NDArray): + assert(source_array.stype != 'default'), \ + "Please use `cast_storage` to create BaseSparseNDArray from an NDArray" + dtype = source_array.dtype if dtype is None else dtype + aux_types = source_array._aux_types if aux_types is None else aux_types + else: + # TODO(haibin/anisub) support creation from scipy object when `_sync_copy_from` is ready + raise NotImplementedError('creating BaseSparseNDArray from ' \ + ' a non-NDArray object is not implemented.') + arr = _empty_sparse_ndarray(source_array.stype, source_array.shape, ctx, dtype, aux_types) + arr[:] = source_array + return arr diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 8bb2c9fb0ea2..81fbe7b24c10 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -21,7 +21,8 @@ pass import mxnet as mx from .context import Context -from .ndarray.ndarray import array, _STORAGE_TYPE_STR_TO_ID +from .ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID +from .ndarray import array from .symbol import Symbol _rng = np.random.RandomState(1234) diff --git a/tests/python/unittest/test_sparse_ndarray.py b/tests/python/unittest/test_sparse_ndarray.py index a7557809e225..eb895cbd198a 100644 --- a/tests/python/unittest/test_sparse_ndarray.py +++ b/tests/python/unittest/test_sparse_ndarray.py @@ -73,12 +73,13 @@ def check_sparse_nd_copy(from_stype, to_stype, shape): shape = rand_shape_2d() shape_3d = rand_shape_3d() - check_sparse_nd_copy('row_sparse', 'row_sparse', shape) - check_sparse_nd_copy('row_sparse', 'default', shape) - check_sparse_nd_copy('default', 'row_sparse', shape) - check_sparse_nd_copy('default', 'csr', shape) + stypes = ['row_sparse', 'csr'] + for stype in stypes: + check_sparse_nd_copy(stype, 'default', shape) + check_sparse_nd_copy('default', stype, shape) check_sparse_nd_copy('row_sparse', 'row_sparse', shape_3d) - + check_sparse_nd_copy('row_sparse', 'default', shape_3d) + check_sparse_nd_copy('default', 'row_sparse', shape_3d) def test_sparse_nd_basic(): def check_sparse_nd_basic_rsp(): @@ -88,6 +89,7 @@ def check_sparse_nd_basic_rsp(): assert(nd._num_aux == 1) assert(nd.indices.dtype == np.int64) assert(nd.stype == 'row_sparse') + check_sparse_nd_basic_rsp() @@ -436,6 +438,8 @@ def test_create_csr(): assert same(csr_created.data.asnumpy(), data.asnumpy()) assert same(csr_created.indptr.asnumpy(), indptr.asnumpy()) assert same(csr_created.indices.asnumpy(), indices.asnumpy()) + csr_copy = mx.nd.array(csr_created) + assert(same(csr_copy.asnumpy(), csr_created.asnumpy())) def test_create_row_sparse(): @@ -451,6 +455,14 @@ def test_create_row_sparse(): assert rsp_created.stype == 'row_sparse' assert same(rsp_created.data.asnumpy(), data.asnumpy()) assert same(rsp_created.indices.asnumpy(), indices.asnumpy()) + rsp_copy = mx.nd.array(rsp_created) + assert(same(rsp_copy.asnumpy(), rsp_created.asnumpy())) + +def test_sparse_nd_empty(): + stypes = ['csr', 'row_sparse', 'default'] + for stype in stypes: + nd = mx.nd.empty((2,2), stype=stype) + assert(nd.stype == stype) if __name__ == '__main__': From 66fed8921d1053dd2a0cb6daed5f9af8ac7f6fb8 Mon Sep 17 00:00:00 2001 From: eric-haibin-lin Date: Wed, 2 Aug 2017 06:17:51 +0000 Subject: [PATCH 2/2] fix print msg in test --- tests/python/unittest/test_sparse_operator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python/unittest/test_sparse_operator.py b/tests/python/unittest/test_sparse_operator.py index aa7349d0dc60..4e97354ea327 100644 --- a/tests/python/unittest/test_sparse_operator.py +++ b/tests/python/unittest/test_sparse_operator.py @@ -265,7 +265,6 @@ def check_sparse_elementwise_sum_with_shape(stype, shape, n): maxdim = 5 for dim in range(2, maxdim): shape = tuple(np.random.randint(5, 10, size=dim)) - print shape check_sparse_elementwise_sum_with_shape('row_sparse', shape, np.random.randint(1, 9))