Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update sparse ndarray api #139

Merged
merged 3 commits into from
Aug 3, 2017
Merged

update sparse ndarray api #139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c6b1bcb
support mx.nd.empty for sparse ndarray
eric-haibin-lin Jul 30, 2017
66fed89
fix print msg in test
eric-haibin-lin Aug 2, 2017
30f23ad
Merge branch 'sparse' into empty
eric-haibin-lin Aug 2, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions python/mxnet/ndarray/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
from . import _internal
from . import op
from .op import CachedOp
from .ndarray import NDArray, array, concatenate, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP
from .ndarray import empty, ones, add, arange, divide, equal, full, greater, greater_equal, imdecode
from .ndarray import NDArray, concatenate, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP
from .ndarray import ones, add, arange, divide, equal, full, greater, greater_equal, imdecode
from .ndarray import lesser, lesser_equal, maximum, minimum, moveaxis, multiply, negative, not_equal
from .ndarray import onehot_encode, power, subtract, true_divide, waitall, _new_empty_handle
from .ndarray_utils import load, save, zeros
from .sparse_ndarray import _ndarray_cls
from .sparse_ndarray import csr, row_sparse, SparseNDArray, todense, RowSparseNDArray, CSRNDArray
from .ndarray_utils import load, save, zeros, empty, array
from .sparse_ndarray import _ndarray_cls, todense
from .sparse_ndarray import csr, row_sparse, BaseSparseNDArray, RowSparseNDArray, CSRNDArray
85 changes: 32 additions & 53 deletions python/mxnet/ndarray/ndarray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -921,7 +921,7 @@ def astype(self, dtype):
>>> y.dtype
<type 'numpy.int32'>
"""
res = empty(self.shape, ctx=self.context, dtype=dtype)
res = _empty_ndarray(self.shape, ctx=self.context, dtype=dtype)
self.copyto(res)
return res

Expand All @@ -942,7 +942,7 @@ def copyto(self, other):

Returns
-------
NDArray
NDArray, CSRNDArray, RowSparseNDArray
The copied array. If ``other`` is an ``NDArray``, then the return value
and ``other`` will point to the same ``NDArray``.

Expand Down Expand Up @@ -1090,39 +1090,6 @@ def onehot_encode(indices, out):
# pylint: enable= no-member, protected-access


def empty(shape, ctx=None, dtype=mx_real_t):
"""Returns a new array of given shape and type, without initializing entries.

Parameters
----------
shape : int or tuple of int
The shape of the empty array.
ctx : Context, optional
An optional device context (default is the current default context).
dtype : str or numpy.dtype, optional
An optional value type (default is `float32`).

Returns
-------
NDArray
A created array.

Examples
--------
>>> mx.nd.empty(1)
<NDArray 1 @cpu(0)>
>>> mx.nd.empty((1,2), mx.gpu(0))
<NDArray 1x2 @gpu(0)>
>>> mx.nd.empty((1,2), mx.gpu(0), 'float16')
<NDArray 1x2 @gpu(0)>
"""
if isinstance(shape, integer_types):
shape = (shape, )
if ctx is None:
ctx = Context.default_ctx
return NDArray(handle=_new_alloc_handle(shape, ctx, False, dtype))


def ones(shape, ctx=None, dtype=None, **kwargs):
"""Returns a new array filled with all ones, with the given shape and type.

Expand Down Expand Up @@ -1190,12 +1157,11 @@ def full(shape, val, ctx=None, dtype=mx_real_t, out=None):
>>> mx.nd.full((1, 2), 2.0, dtype='float16').asnumpy()
array([[ 2., 2.]], dtype=float16)
"""
out = empty(shape, ctx, dtype) if out is None else out
out = _empty_ndarray(shape, ctx, dtype) if out is None else out
out[:] = val
return out


def array(source_array, ctx=None, dtype=None):
def _array(source_array, ctx=None, dtype=None):
"""Creates an array from any object exposing the array interface.

Parameters
Expand All @@ -1213,18 +1179,6 @@ def array(source_array, ctx=None, dtype=None):
-------
NDArray
An `NDArray` with the same contents as the `source_array`.

Examples
--------
>>> import numpy as np
>>> mx.nd.array([1, 2, 3])
<NDArray 3 @cpu(0)>
>>> mx.nd.array([[1, 2], [3, 4]])
<NDArray 2x2 @cpu(0)>
>>> mx.nd.array(np.zeros((3, 2)))
<NDArray 3x2 @cpu(0)>
>>> mx.nd.array(np.zeros((3, 2)), mx.gpu(0))
<NDArray 3x2 @gpu(0)>
"""
if isinstance(source_array, NDArray):
dtype = source_array.dtype if dtype is None else dtype
Expand All @@ -1235,11 +1189,10 @@ def array(source_array, ctx=None, dtype=None):
source_array = np.array(source_array, dtype=dtype)
except:
raise TypeError('source_array must be array like object')
arr = empty(source_array.shape, ctx, dtype)
arr = _empty_ndarray(source_array.shape, ctx, dtype)
arr[:] = source_array
return arr


def moveaxis(tensor, source, destination):
"""Moves the `source` axis into the `destination` position
while leaving the other axes in their original order
Expand Down Expand Up @@ -2289,7 +2242,7 @@ def concatenate(arrays, axis=0, always_copy=True):
assert shape_rest2 == arr.shape[axis+1:]
assert dtype == arr.dtype
ret_shape = shape_rest1 + (shape_axis,) + shape_rest2
ret = empty(ret_shape, ctx=arrays[0].context, dtype=dtype)
ret = _empty_ndarray(ret_shape, ctx=arrays[0].context, dtype=dtype)

idx = 0
begin = [0 for _ in ret_shape]
Expand Down Expand Up @@ -2386,3 +2339,29 @@ def _zeros_ndarray(shape, ctx=None, dtype=None, **kwargs):
# pylint: disable= no-member, protected-access
return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, **kwargs)
# pylint: enable= no-member, protected-access

def _empty_ndarray(shape, ctx=None, dtype=None):
"""Returns a new array of given shape and type, without initializing entries.

Parameters
----------
shape : int or tuple of int
The shape of the empty array.
ctx : Context, optional
An optional device context (default is the current default context).
dtype : str or numpy.dtype, optional
An optional value type (default is `float32`).

Returns
-------
NDArray
A created array.

"""
if isinstance(shape, int):
shape = (shape, )
if ctx is None:
ctx = Context.default_ctx
if dtype is None:
dtype = mx_real_t
return NDArray(handle=_new_alloc_handle(shape, ctx, False, dtype))
101 changes: 92 additions & 9 deletions python/mxnet/ndarray/ndarray_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
# coding: utf-8
"""Utility functions for NDArray and SparseNDArray."""
"""Utility functions for NDArray and BaseSparseNDArray."""
import ctypes

from ..base import _LIB, check_call, py_str, c_str, string_types, mx_uint, NDArrayHandle, c_array
from .ndarray import NDArray, _zeros_ndarray
from .sparse_ndarray import _ndarray_cls, _zeros_sparse_ndarray
from .ndarray import NDArray, _zeros_ndarray, _empty_ndarray, _array
from .sparse_ndarray import _zeros_sparse_ndarray, _empty_sparse_ndarray, _sparse_array
from .sparse_ndarray import _ndarray_cls


def zeros(shape, ctx=None, dtype=None, stype=None, aux_types=None, **kwargs):
Expand All @@ -21,12 +22,12 @@ def zeros(shape, ctx=None, dtype=None, stype=None, aux_types=None, **kwargs):
stype: string, optional
The storage type of the empty array, such as 'row_sparse', 'csr', etc
aux_types: list of numpy.dtype, optional
An optional type for the aux data for SparseNDArray (default values depends
on the storage type)
An optional type for the aux data for the BaseSparseNDArray (default values
depends on the storage type)

Returns
-------
SparseNDArray
NDArray, CSRNDArray or RowSparseNDArray
A created array
Examples
--------
Expand All @@ -36,11 +37,90 @@ def zeros(shape, ctx=None, dtype=None, stype=None, aux_types=None, **kwargs):
array([[ 0., 0.]], dtype=float16)
"""

if stype is None:
if stype is None or stype == 'default':
return _zeros_ndarray(shape, ctx, dtype, **kwargs)
else:
return _zeros_sparse_ndarray(stype, shape, ctx, dtype, aux_types, **kwargs)

def empty(shape, ctx=None, dtype=None, stype=None, aux_types=None):
"""Returns a new array of given shape and type, without initializing entries.

Parameters
----------
shape : int or tuple of int
The shape of the empty array.
ctx : Context, optional
An optional device context (default is the current default context).
dtype : str or numpy.dtype, optional
An optional value type (default is `float32`).
stype : str, optional
An optional storage type (default is `default`).
aux_types: list of numpy.dtype, optional
An optional type for the aux data for the BaseSparseNDArray (default values depends
on the storage type)

Returns
-------
NDArray, CSRNDArray or RowSparseNDArray
A created array.

Examples
--------
>>> mx.nd.empty(1)
<NDArray 1 @cpu(0)>
>>> mx.nd.empty((1,2), mx.gpu(0))
<NDArray 1x2 @gpu(0)>
>>> mx.nd.empty((1,2), mx.gpu(0), 'float16')
<NDArray 1x2 @gpu(0)>
>>> mx.nd.empty((1,2), stype='csr')
<CSRNDArray 1x2 @cpu(0)>
"""
if stype is None or stype == 'default':
return _empty_ndarray(shape, ctx, dtype)
else:
return _empty_sparse_ndarray(stype, shape, ctx, dtype, aux_types)

def array(source_array, ctx=None, dtype=None, aux_types=None):
"""Creates an array from any object exposing the array interface.

Parameters
----------
source_array : array_like
An object exposing the array interface, an object whose `__array__`
method returns an array, or any (nested) sequence.
ctx : Context, optional
Device context (default is the current default context).
dtype : str or numpy.dtype, optional
The data type of the output array. The default dtype is ``source_array.dtype``
if `source_array` is an `NDArray`, `float32` otherwise.
aux_types: list of numpy.dtype, optional
An optional type for the aux data for the BaseSparseNDArray (default values
depends on the storage type)

Returns
-------
NDArray, RowSparseNDArray or CSRNDArray
An array with the same contents as the `source_array`.

Examples
--------
>>> import numpy as np
>>> mx.nd.array([1, 2, 3])
<NDArray 3 @cpu(0)>
>>> mx.nd.array([[1, 2], [3, 4]])
<NDArray 2x2 @cpu(0)>
>>> mx.nd.array(np.zeros((3, 2)))
<NDArray 3x2 @cpu(0)>
>>> mx.nd.array(np.zeros((3, 2)), mx.gpu(0))
<NDArray 3x2 @gpu(0)>
>>> mx.nd.array(mx.nd.zeros((3, 2), stype='row_sparse'))
<RowSparseNDArray 3x2 @cpu(0)>
"""
# TODO(haibin/anisub) Check if input is scipy.sparse object with `scipy.sparse.issparse`
if isinstance(source_array, NDArray) and source_array.stype != 'default':
return _sparse_array(source_array, ctx=ctx, dtype=dtype, aux_types=aux_types)
else:
return _array(source_array, ctx=ctx, dtype=dtype)

def load(fname):
"""Loads an array from file.
Expand All @@ -54,7 +134,8 @@ def load(fname):

Returns
-------
list of NDArray or dict of str to NDArray
list of NDArray, RowSparseNDArray or CSRNDArray, or \
dict of str to NDArray, RowSparseNDArray or CSRNDArray
Loaded data.
"""
if not isinstance(fname, string_types):
Expand Down Expand Up @@ -90,7 +171,9 @@ def save(fname, data):
----------
fname : str
The filename.
data : ``NDArray``, list of ``NDArray` or dict of str to ``NDArray``
data : NDArray, RowSparseNDArray or CSRNDArray, \
or list of NDArray, RowSparseNDArray or CSRNDArray, \
or dict of str to NDArray, RowSparseNDArray or CSRNDArray
The data to save.

Examples
Expand Down
Loading