From b219f1cf00fa69ddc42a141aa371ddc8eb6923a6 Mon Sep 17 00:00:00 2001 From: Miles Date: Sat, 29 Apr 2023 06:57:45 +0200 Subject: [PATCH] Accept anything with Buffer Protocol, remove numpy (#100) - Adds support for anything which implements the buffer protocol - As a result, removes need for numpy and PyBytes and PyByteArray implementations in BytesType variants --- .github/workflows/CI.yml | 5 - Cargo.lock | 80 +---------- Cargo.toml | 3 +- src/io.rs | 297 +++++++++++++++------------------------ src/lib.rs | 65 +++------ tests/test_no_numpy.py | 23 --- tests/test_variants.py | 8 +- 7 files changed, 137 insertions(+), 344 deletions(-) delete mode 100644 tests/test_no_numpy.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c498d2e2..26e28003 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -122,11 +122,6 @@ jobs: pip install cramjam --no-index --find-links dist --force-reinstall pip install -r dev-requirements.txt make test - - name: Test no numpy installed works - if: matrix.target == 'x86_64' - run: | - pip uninstall numpy -y - python -m pytest tests/test_no_numpy.py - name: Upload wheels uses: actions/upload-artifact@v3 with: diff --git a/Cargo.lock b/Cargo.lock index d4583aff..d9631ea2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,13 +94,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cramjam" -version = "2.7.0-rc1" +version = "2.7.0-rc2" dependencies = [ "brotli", "bzip2", "flate2", "lz4", - "numpy", "pyo3", "snap", "zstd", @@ -176,15 +175,6 @@ dependencies = [ "libc", ] -[[package]] -name = "matrixmultiply" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add85d4dd35074e6fedc608f8c8f513a3548619a9024b751949ef0e8e45a4d84" -dependencies = [ - "rawpointer", -] - [[package]] name = "memoffset" version = "0.8.0" @@ -203,62 +193,6 @@ dependencies = [ "adler", ] -[[package]] -name = "ndarray" -version = "0.15.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "rawpointer", -] - -[[package]] -name = "num-complex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" -dependencies = [ - "autocfg", -] - -[[package]] -name = "numpy" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b0fee4571867d318651c24f4a570c3f18408cf95f16ccb576b3ce85496a46e" -dependencies = [ - "libc", - "ndarray", - "num-complex", - "num-integer", - "num-traits", - "pyo3", - "rustc-hash", -] - [[package]] name = "once_cell" version = "1.17.1" @@ -372,12 +306,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - [[package]] name = "redox_syscall" version = "0.2.16" @@ -387,12 +315,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "scopeguard" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index dec2cccb..3208d827 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cramjam" -version = "2.7.0-rc1" +version = "2.7.0-rc2" authors = ["Miles Granger "] edition = "2018" license = "MIT" @@ -28,4 +28,3 @@ bzip2 = "^0.4" lz4 = "^1" flate2 = "1.0.25" zstd = "0.11.1+zstd.1.5.2" -numpy = "^0.18" diff --git a/src/io.rs b/src/io.rs index 09a97fc0..98e9065a 100644 --- a/src/io.rs +++ b/src/io.rs @@ -2,15 +2,17 @@ //! which wrap native Python objects to provide additional functionality //! or tighter integration with de/compression algorithms. //! +use std::convert::TryFrom; use std::fs::{File, OpenOptions}; use std::io::{copy, Cursor, Read, Seek, SeekFrom, Write}; use std::os::raw::c_int; use crate::exceptions::CompressionError; use crate::BytesType; -use numpy::PyArray1; +use pyo3::buffer::{Element, PyBuffer}; +use pyo3::exceptions::PyBufferError; use pyo3::prelude::*; -use pyo3::types::{PyByteArray, PyBytes}; +use pyo3::types::PyBytes; use pyo3::{ffi, AsPyPointer}; use std::path::PathBuf; @@ -19,179 +21,6 @@ pub(crate) trait AsBytes { fn as_bytes_mut(&mut self) -> &mut [u8]; } -/// Internal wrapper for `numpy.array`/`PyArray1`, to provide Read + Write and other traits -pub struct RustyNumpyArray<'a> { - pub(crate) inner: &'a PyArray1, - pub(crate) cursor: Cursor<&'a mut [u8]>, -} -impl<'a> AsBytes for RustyNumpyArray<'a> { - fn as_bytes(&self) -> &[u8] { - self.cursor.get_ref().as_ref() - } - fn as_bytes_mut(&mut self) -> &mut [u8] { - self.cursor.get_mut() - } -} -impl<'a> RustyNumpyArray<'a> { - pub(crate) fn as_bytes(&self) -> &[u8] { - unsafe { self.inner.as_slice().unwrap() } - } -} -impl<'a> From<&'a PyArray1> for RustyNumpyArray<'a> { - fn from(inner: &'a PyArray1) -> Self { - Self { - inner, - cursor: Cursor::new(unsafe { inner.as_slice_mut().unwrap() }), - } - } -} -impl<'a> FromPyObject<'a> for RustyNumpyArray<'a> { - fn extract(ob: &'a PyAny) -> PyResult { - let pybytes: &PyArray1 = ob.extract()?; - Ok(Self::from(pybytes)) - } -} -impl<'a> ToPyObject for RustyNumpyArray<'a> { - fn to_object(&self, py: Python<'_>) -> PyObject { - self.inner.to_object(py) - } -} -impl<'a> Write for RustyNumpyArray<'a> { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.cursor.write(buf) - } - fn flush(&mut self) -> std::io::Result<()> { - self.cursor.flush() - } -} -impl<'a> Read for RustyNumpyArray<'a> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.cursor.read(buf) - } -} - -impl<'a> Seek for RustyNumpyArray<'a> { - fn seek(&mut self, pos: SeekFrom) -> std::io::Result { - self.cursor.seek(pos) - } -} - -/// Internal wrapper for `bytes`/`PyBytes`, to provide Read + Write and other traits -pub struct RustyPyBytes<'a> { - pub(crate) inner: &'a PyBytes, - pub(crate) cursor: Cursor<&'a mut [u8]>, -} -impl<'a> AsBytes for RustyPyBytes<'a> { - fn as_bytes(&self) -> &[u8] { - self.inner.as_bytes() - } - fn as_bytes_mut(&mut self) -> &mut [u8] { - self.cursor.get_mut() - } -} -impl<'a> From<&'a PyBytes> for RustyPyBytes<'a> { - fn from(inner: &'a PyBytes) -> Self { - let ptr = inner.as_bytes().as_ptr(); - Self { - inner, - cursor: Cursor::new(unsafe { std::slice::from_raw_parts_mut(ptr as *mut _, inner.as_bytes().len()) }), - } - } -} -impl<'a> FromPyObject<'a> for RustyPyBytes<'a> { - fn extract(ob: &'a PyAny) -> PyResult { - let pybytes: &PyBytes = ob.extract()?; - Ok(Self::from(pybytes)) - } -} -impl<'a> ToPyObject for RustyPyBytes<'a> { - fn to_object(&self, py: Python<'_>) -> PyObject { - self.inner.to_object(py) - } -} -impl<'a> Read for RustyPyBytes<'a> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.cursor.read(buf) - } -} -impl<'a> Write for RustyPyBytes<'a> { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.cursor.write(buf) - } - fn flush(&mut self) -> std::io::Result<()> { - self.cursor.flush() - } -} -impl<'a> Seek for RustyPyBytes<'a> { - fn seek(&mut self, style: SeekFrom) -> std::io::Result { - self.cursor.seek(style) - } -} - -/// Internal wrapper for `bytearray`/`PyByteArray`, to provide Read + Write and other traits -pub struct RustyPyByteArray<'a> { - pub(crate) inner: &'a PyByteArray, - pub(crate) cursor: Cursor<&'a mut [u8]>, -} -impl<'a> AsBytes for RustyPyByteArray<'a> { - fn as_bytes(&self) -> &[u8] { - self.cursor.get_ref() - } - fn as_bytes_mut(&mut self) -> &mut [u8] { - self.cursor.get_mut() - } -} -impl<'a> From<&'a PyByteArray> for RustyPyByteArray<'a> { - fn from(inner: &'a PyByteArray) -> Self { - Self { - inner, - cursor: Cursor::new(unsafe { inner.as_bytes_mut() }), - } - } -} -impl<'a> FromPyObject<'a> for RustyPyByteArray<'a> { - fn extract(ob: &'a PyAny) -> PyResult { - let pybytes: &PyByteArray = ob.extract()?; - Ok(Self::from(pybytes)) - } -} -impl<'a> ToPyObject for RustyPyByteArray<'a> { - fn to_object(&self, py: Python<'_>) -> PyObject { - self.inner.to_object(py) - } -} -impl<'a> Write for RustyPyByteArray<'a> { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - if (self.cursor.position() as usize + buf.len()) > self.inner.len() { - let previous_pos = self.cursor.position(); - self.inner.resize(self.cursor.position() as usize + buf.len()).unwrap(); - self.cursor = Cursor::new(unsafe { self.inner.as_bytes_mut() }); - self.cursor.set_position(previous_pos); - } - self.cursor.write(buf) - } - fn flush(&mut self) -> std::io::Result<()> { - if self.inner.len() != self.cursor.position() as usize { - let prev_pos = self.cursor.position(); - self.inner.resize(self.cursor.position() as usize).unwrap(); - self.cursor = Cursor::new(unsafe { self.inner.as_bytes_mut() }); - self.cursor.set_position(prev_pos); - } - Ok(()) - } -} -impl<'a> Read for RustyPyByteArray<'a> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.cursor.read(buf) - } -} - -impl<'a> Seek for RustyPyByteArray<'a> { - fn seek(&mut self, pos: SeekFrom) -> std::io::Result { - self.cursor.seek(pos) - } -} - /// A native Rust file-like object. Reading and writing takes place /// through the Rust implementation, allowing access to the underlying /// bytes in Python. @@ -327,7 +156,7 @@ impl RustyFile { Some(path) => path.to_string(), None => self.path.to_string_lossy().to_string(), }; - let repr = format!("cramjam.File(path={}, len={:?})", path, self.len()?); + let repr = format!("cramjam.File", path, self.len()?); Ok(repr) } fn __bool__(&self) -> PyResult { @@ -338,6 +167,97 @@ impl RustyFile { } } +/// Internal wrapper to PyBuffer, not exposed thru API +/// used only for impl of Read/Write +pub struct PythonBuffer { + pub(crate) inner: PyBuffer, + pub(crate) pos: usize, +} +impl PythonBuffer { + /// Reset the read/write position of cursor + pub fn reset_position(&mut self) { + self.pos = 0; + } + /// Explicitly set the position of the cursor + pub fn set_position(&mut self, pos: usize) { + self.pos = pos; + } + /// Get the current position of the cursor + pub fn position(&self) -> usize { + self.pos + } + /// Is the Python buffer readonly + pub fn readonly(&self) -> bool { + self.inner.readonly() + } + /// Get the underlying buffer as a slice of bytes + pub fn as_slice(&self) -> &[u8] { + unsafe { std::slice::from_raw_parts(self.inner.buf_ptr() as *const u8, self.inner.len_bytes()) } + } + /// Get the underlying buffer as a mutable slice of bytes + pub fn as_slice_mut(&mut self) -> PyResult<&mut [u8]> { + // TODO: For v3 release, add self.readonly check; bytes is readonly but + // v1 and v2 releases have not treated it as such. + Ok(unsafe { std::slice::from_raw_parts_mut(self.inner.buf_ptr() as *mut u8, self.inner.len_bytes()) }) + } +} + +impl<'py, T: Element> FromPyObject<'py> for PythonBuffer { + fn extract(obj: &'py PyAny) -> PyResult { + let buf = PyBuffer::get(obj)?; + PythonBuffer::try_from(buf) + } +} + +impl TryFrom> for PythonBuffer { + type Error = PyErr; + fn try_from(buf: PyBuffer) -> Result { + if !buf.is_c_contiguous() { + Err(PyBufferError::new_err("Buffer is not C contiguous")) + } else if buf.dimensions() != 1 { + Err(PyBufferError::new_err("Buffer is not 1 dimensional")) + } else { + Ok(Self { inner: buf, pos: 0 }) + } + } +} + +impl Read for PythonBuffer { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let slice = self.as_slice(); + if self.pos < slice.len() { + let nbytes = (&slice[self.pos..]).read(buf)?; + self.pos += nbytes; + Ok(nbytes) + } else { + Ok(0) + } + } +} + +impl Write for PythonBuffer { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let pos = self.position(); + let slice = self + .as_slice_mut() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; + let len = slice.len(); + + if pos < slice.len() { + let nbytes = std::cmp::min(len - pos, buf.len()); + slice[pos..pos + nbytes].copy_from_slice(&buf[..nbytes]); + self.pos += nbytes; + Ok(nbytes) + } else { + Ok(0) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + /// A native Rust file-like object. Reading and writing takes place /// through the Rust implementation, allowing access to the underlying /// bytes in Python. @@ -462,7 +382,7 @@ impl RustyBuffer { py.allow_threads(|| self.inner.get_ref().windows(bytes.len()).any(|w| w == bytes)) } fn __repr__(&self) -> String { - format!("cramjam.Buffer(len={:?})", self.len()) + format!("cramjam.Buffer", self.len()) } fn __bool__(&self) -> bool { self.len() > 0 @@ -483,7 +403,7 @@ impl RustyBuffer { (*view).buf = bytes.as_ptr() as *mut std::os::raw::c_void; (*view).len = bytes.len() as isize; - (*view).readonly = 1; + (*view).readonly = 0; (*view).itemsize = 1; (*view).format = std::ptr::null_mut(); @@ -512,14 +432,9 @@ impl RustyBuffer { fn write(input: &mut BytesType, output: &mut W) -> std::io::Result { let result = match input { + BytesType::RustyBuffer(buf) => copy(&mut buf.borrow_mut().inner, output)?, BytesType::RustyFile(data) => copy(&mut data.borrow_mut().inner, output)?, - BytesType::RustyBuffer(data) => copy(&mut data.borrow_mut().inner, output)?, - BytesType::ByteArray(data) => copy(data, output)?, - BytesType::NumpyArray(array) => copy(array, output)?, - BytesType::Bytes(data) => { - let buffer = data.as_bytes(); - copy(&mut Cursor::new(buffer), output)? - } + BytesType::PyBuffer(buf) => copy(buf, output)?, }; Ok(result) } @@ -548,6 +463,18 @@ impl Seek for RustyFile { self.inner.seek(pos) } } +impl Seek for PythonBuffer { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + let len = self.inner.len_bytes(); + let current = self.position(); + match pos { + SeekFrom::Start(n) => self.set_position(n as usize), + SeekFrom::End(n) => self.set_position((len as i64 - n) as usize), + SeekFrom::Current(n) => self.set_position((current as i64 + n) as usize), + } + Ok(self.position() as _) + } +} impl Write for RustyBuffer { fn write(&mut self, buf: &[u8]) -> std::io::Result { self.inner.write(buf) diff --git a/src/lib.rs b/src/lib.rs index 9f4c0d2c..5a1413de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,9 +61,10 @@ pub mod lz4; pub mod snappy; pub mod zstd; +use io::{PythonBuffer, RustyBuffer}; use pyo3::prelude::*; -use crate::io::{AsBytes, RustyBuffer, RustyFile, RustyNumpyArray, RustyPyByteArray, RustyPyBytes}; +use crate::io::{AsBytes, RustyFile}; use exceptions::{CompressionError, DecompressionError}; use std::io::{Read, Seek, SeekFrom, Write}; @@ -72,34 +73,26 @@ use std::io::{Read, Seek, SeekFrom, Write}; /// the documentation to see what types are acceptable for de/compression functions. #[derive(FromPyObject)] pub enum BytesType<'a> { - /// `bytes` - #[pyo3(transparent, annotation = "bytes")] - Bytes(RustyPyBytes<'a>), - /// `bytearray` - #[pyo3(transparent, annotation = "bytearray")] - ByteArray(RustyPyByteArray<'a>), - /// [`cramjam.File`](io/struct.RustyFile.html) - #[pyo3(transparent, annotation = "File")] - RustyFile(&'a PyCell), /// [`cramjam.Buffer`](io/struct.RustyBuffer.html) #[pyo3(transparent, annotation = "Buffer")] RustyBuffer(&'a PyCell), - /// `numpy.array` with `dtype=np.uint8` - #[pyo3(transparent, annotation = "numpy")] - NumpyArray(RustyNumpyArray<'a>), + /// [`cramjam.File`](io/struct.RustyFile.html) + #[pyo3(transparent, annotation = "File")] + RustyFile(&'a PyCell), + /// `object` implementing the Buffer Protocol + #[pyo3(transparent, annotation = "pybuffer")] + PyBuffer(PythonBuffer), } impl<'a> AsBytes for BytesType<'a> { fn as_bytes(&self) -> &[u8] { match self { - BytesType::Bytes(b) => b.as_bytes(), - BytesType::ByteArray(b) => b.as_bytes(), - BytesType::NumpyArray(b) => b.as_bytes(), BytesType::RustyBuffer(b) => { let py_ref = b.borrow(); let bytes = py_ref.as_bytes(); unsafe { std::slice::from_raw_parts(bytes.as_ptr(), bytes.len()) } } + BytesType::PyBuffer(b) => b.as_slice(), BytesType::RustyFile(b) => { let py_ref = b.borrow(); let bytes = py_ref.as_bytes(); @@ -109,14 +102,12 @@ impl<'a> AsBytes for BytesType<'a> { } fn as_bytes_mut(&mut self) -> &mut [u8] { match self { - BytesType::Bytes(b) => b.as_bytes_mut(), - BytesType::ByteArray(b) => b.as_bytes_mut(), - BytesType::NumpyArray(b) => b.as_bytes_mut(), BytesType::RustyBuffer(b) => { let mut py_ref = b.borrow_mut(); let bytes = py_ref.as_bytes_mut(); unsafe { std::slice::from_raw_parts_mut(bytes.as_mut_ptr(), bytes.len()) } } + BytesType::PyBuffer(b) => b.as_slice_mut().unwrap(), BytesType::RustyFile(b) => { let mut py_ref = b.borrow_mut(); let bytes = py_ref.as_bytes_mut(); @@ -129,41 +120,35 @@ impl<'a> AsBytes for BytesType<'a> { impl<'a> Write for BytesType<'a> { fn write(&mut self, buf: &[u8]) -> std::io::Result { let result = match self { - BytesType::RustyFile(out) => out.borrow_mut().inner.write(buf)?, BytesType::RustyBuffer(out) => out.borrow_mut().inner.write(buf)?, - BytesType::ByteArray(out) => out.write(buf)?, - BytesType::NumpyArray(out) => out.write(buf)?, - BytesType::Bytes(out) => out.write(buf)?, + BytesType::RustyFile(out) => out.borrow_mut().inner.write(buf)?, + BytesType::PyBuffer(out) => out.write(buf)?, }; Ok(result) } fn flush(&mut self) -> std::io::Result<()> { match self { - BytesType::RustyFile(f) => f.borrow_mut().flush(), BytesType::RustyBuffer(b) => b.borrow_mut().flush(), - BytesType::ByteArray(_) | BytesType::Bytes(_) | BytesType::NumpyArray(_) => Ok(()), + BytesType::RustyFile(f) => f.borrow_mut().flush(), + BytesType::PyBuffer(_) => Ok(()), } } } impl<'a> Read for BytesType<'a> { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { match self { - BytesType::RustyFile(data) => data.borrow_mut().inner.read(buf), BytesType::RustyBuffer(data) => data.borrow_mut().inner.read(buf), - BytesType::ByteArray(data) => data.read(buf), - BytesType::NumpyArray(array) => array.read(buf), - BytesType::Bytes(data) => data.read(buf), + BytesType::RustyFile(data) => data.borrow_mut().inner.read(buf), + BytesType::PyBuffer(data) => data.read(buf), } } } impl<'a> Seek for BytesType<'a> { fn seek(&mut self, style: SeekFrom) -> std::io::Result { match self { - BytesType::RustyFile(f) => f.borrow_mut().inner.seek(style), BytesType::RustyBuffer(b) => b.borrow_mut().inner.seek(style), - BytesType::ByteArray(a) => a.seek(style), - BytesType::NumpyArray(a) => a.seek(style), - BytesType::Bytes(b) => b.seek(style), + BytesType::RustyFile(f) => f.borrow_mut().inner.seek(style), + BytesType::PyBuffer(buf) => buf.seek(style), } } } @@ -174,18 +159,6 @@ impl<'a> BytesType<'a> { } } -impl<'a> IntoPy for BytesType<'a> { - fn into_py(self, py: Python) -> PyObject { - match self { - Self::Bytes(bytes) => bytes.inner.into(), - Self::ByteArray(byte_array) => byte_array.inner.into(), - Self::RustyFile(file) => file.to_object(py), - Self::RustyBuffer(buffer) => buffer.into_py(py), - Self::NumpyArray(array) => array.to_object(py), - } - } -} - /// Macro for generating the implementation of de/compression against a variant interface #[macro_export] macro_rules! generic { @@ -236,7 +209,7 @@ macro_rules! generic { $py.allow_threads(|| { $op(f_in, &mut buf_out $(, $level)? ) }) - } + }, _ => { let bytes_out = $output.as_bytes_mut(); $py.allow_threads(|| { diff --git a/tests/test_no_numpy.py b/tests/test_no_numpy.py deleted file mode 100644 index 82ed9e51..00000000 --- a/tests/test_no_numpy.py +++ /dev/null @@ -1,23 +0,0 @@ -import pytest -import cramjam - - -@pytest.mark.parametrize("obj", (bytes, bytearray, cramjam.Buffer, cramjam.File)) -@pytest.mark.parametrize( - "variant_str", ("snappy", "brotli", "lz4", "gzip", "deflate", "zstd") -) -def test_no_numpy_installed(tmpdir, obj, variant_str): - """ - These operations should work even when numpy is not installed - """ - if cramjam.File == obj: - data = obj(str(tmpdir.join("tmp.txt"))) - data.write(b"data") - data.seek(0) - else: - data = obj(b"data") - - variant = getattr(cramjam, variant_str) - compressed = variant.compress(data) - decompressed = variant.decompress(compressed) - assert decompressed.read() == b"data" diff --git a/tests/test_variants.py b/tests/test_variants.py index 47f38868..e786c6a7 100644 --- a/tests/test_variants.py +++ b/tests/test_variants.py @@ -56,10 +56,10 @@ def test_variants_raise_exception(variant_str): @pytest.mark.parametrize( - "input_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File) + "input_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File, memoryview) ) @pytest.mark.parametrize( - "output_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File) + "output_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File, memoryview) ) @pytest.mark.parametrize("variant_str", VARIANTS) @given(raw_data=st.binary()) @@ -113,10 +113,10 @@ def test_variants_compress_into( @pytest.mark.parametrize( - "input_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File) + "input_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File, memoryview) ) @pytest.mark.parametrize( - "output_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File) + "output_type", (bytes, bytearray, "numpy", cramjam.Buffer, cramjam.File, memoryview) ) @pytest.mark.parametrize("variant_str", VARIANTS) @given(raw_data=st.binary())