Skip to content

Commit

Permalink
[Python]: add bindings for additional Buffer class non-CPU methods
Browse files Browse the repository at this point in the history
  • Loading branch information
anjakefala committed Aug 1, 2024
1 parent 0dec116 commit 3927fe7
Show file tree
Hide file tree
Showing 5 changed files with 43 additions and 1 deletion.
4 changes: 3 additions & 1 deletion python/pyarrow/device.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ cdef class MemoryManager(_Weakrefable):
self.init(mm)
return self

cdef inline shared_ptr[CMemoryManager] unwrap(self) nogil:
return self.memory_manager

def __repr__(self):
return "<pyarrow.MemoryManager device: {}>".format(
frombytes(self.memory_manager.get().device().get().ToString())
Expand All @@ -152,7 +155,6 @@ cdef class MemoryManager(_Weakrefable):
"""
return self.memory_manager.get().is_cpu()


def default_cpu_memory_manager():
"""
Return the default CPU MemoryManager instance.
Expand Down
1 change: 1 addition & 0 deletions python/pyarrow/includes/libarrow.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
shared_ptr[CDevice] device()
const shared_ptr[CMemoryManager] memory_manager()
CDeviceAllocationType device_type()
CResult[shared_ptr[CBuffer]] Copy(shared_ptr[CBuffer] source, const shared_ptr[CMemoryManager]& to) const

CResult[shared_ptr[CBuffer]] SliceBufferSafe(
const shared_ptr[CBuffer]& buffer, int64_t offset)
Expand Down
21 changes: 21 additions & 0 deletions python/pyarrow/io.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -1446,6 +1446,27 @@ cdef class Buffer(_Weakrefable):
"""
return _wrap_device_allocation_type(self.buffer.get().device_type())

def copy(self, MemoryManager mm):
"""
The buffer contents will be copied into a new buffer allocated by the
given MemoryManager. This function supports cross-device copies.
Parameters
---------
MemoryManager
Memory manager used to allocate the new buffer.
Returns
-------
Buffer
"""
cdef:
shared_ptr[CBuffer] c_buffer

c_buffer = GetResultValue(self.buffer.get().Copy(self.buffer, mm.unwrap()))
return pyarrow_wrap_buffer(c_buffer)


@property
def parent(self):
cdef shared_ptr[CBuffer] parent_buf = self.buffer.get().parent()
Expand Down
2 changes: 2 additions & 0 deletions python/pyarrow/lib.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,8 @@ cdef class MemoryManager(_Weakrefable):
@staticmethod
cdef wrap(const shared_ptr[CMemoryManager]& mm)

cdef inline shared_ptr[CMemoryManager] unwrap(self) nogil


cdef class Buffer(_Weakrefable):
cdef:
Expand Down
16 changes: 16 additions & 0 deletions python/pyarrow/tests/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,22 @@ def make_random_buffer(size, target='host'):
return arr, dbuf
raise ValueError('invalid target value')

def test_copy_from_buffer():
arr, buf = make_random_buffer(128)
cudabuf = global_context.buffer_from_data(buf)

mm2 = global_context1.memory_manager
buf2 = cudabuf.copy(mm2)
cudabuf2 = cuda.CudaBuffer.from_buffer(buf2)
cudabuf2.size == cudabuf.size

arr2 = np.frombuffer(cudabuf2.copy_to_host(), dtype=np.uint8)
np.testing.assert_equal(arr, arr2)

assert cudabuf2.memory_manager.device == mm2.device




@pytest.mark.parametrize("size", [0, 1, 1000])
def test_context_device_buffer(size):
Expand Down

0 comments on commit 3927fe7

Please sign in to comment.