diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d1baf451..14cd903e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -203,12 +203,10 @@ jobs: - '3.12' target: [aarch64, armv7, s390x, ppc64le] include: - - python: pypy3.7 - target: aarch64 - - python: pypy3.8 - target: aarch64 - python: pypy3.9 target: aarch64 + - python: pypy3.10 + target: aarch64 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 @@ -269,9 +267,8 @@ jobs: strategy: matrix: python: - - pypy-3.7 - - pypy-3.8 - pypy-3.9 + - pypy-3.10 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 @@ -301,7 +298,8 @@ jobs: - name: Python UnitTest - cramjam-python run: | pip install cramjam --no-index --find-links dist - pypy -c "import cramjam" + pip install .[dev] + python -m pytest tests -v - name: Python cramjam-cli test run: | pip install cramjam-cli --no-index --find-links dist @@ -316,7 +314,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: [ pypy-3.7, pypy-3.8, pypy-3.9 ] + python-version: [ pypy-3.9, pypy-3.10 ] steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 @@ -352,7 +350,10 @@ jobs: pip install cramjam --no-index --find-links dist pip install cramjam-cli --no-index --find-links dist - name: Python Import test - run: pypy -c "import cramjam" + run: | + pip install cramjam --no-index --find-links dist + pip install .[dev] + python -m pytest tests -v - name: Python cramjam-cli test run: cramjam-cli --help - name: Upload wheels diff --git a/cramjam-python/src/io.rs b/cramjam-python/src/io.rs index d9561204..6d7527e5 100644 --- a/cramjam-python/src/io.rs +++ b/cramjam-python/src/io.rs @@ -18,7 +18,7 @@ use std::path::PathBuf; pub(crate) trait AsBytes { fn as_bytes(&self) -> &[u8]; - fn as_bytes_mut(&mut self) -> &mut [u8]; + fn as_bytes_mut(&mut self) -> PyResult<&mut [u8]>; } /// A native Rust file-like object. Reading and writing takes place @@ -49,7 +49,7 @@ impl AsBytes for RustyFile { entire file into memory; consider using cramjam.Buffer" ) } - fn as_bytes_mut(&mut self) -> &mut [u8] { + fn as_bytes_mut(&mut self) -> PyResult<&mut [u8]> { unimplemented!( "Converting a File to bytes is not supported, as it'd require reading the \ entire file into memory; consider using cramjam.Buffer" @@ -173,6 +173,8 @@ impl RustyFile { pub struct PythonBuffer { pub(crate) inner: std::pin::Pin>, pub(crate) pos: usize, + #[cfg(PyPy)] + pub(crate) owner: PyObject, } // PyBuffer is thread-safe: the shape of the buffer is immutable while a Py_buffer exists. // Accessing the buffer contents is protected using the GIL. @@ -194,7 +196,7 @@ impl PythonBuffer { } /// Is the Python buffer readonly pub fn readonly(&self) -> bool { - self.inner.readonly != 0 + self.inner.readonly == 1 } /// Get the underlying buffer as a slice of bytes pub fn as_slice(&self) -> &[u8] { @@ -202,13 +204,24 @@ impl PythonBuffer { } /// Get the underlying buffer as a mutable slice of bytes pub fn as_slice_mut(&mut self) -> PyResult<&mut [u8]> { - // TODO: For v3 release, add self.readonly check; bytes is readonly but - // v1 and v2 releases have not treated it as such. + #[cfg(PyPy)] + { + Python::with_gil(|py| { + let is_memoryview = unsafe { ffi::PyMemoryView_Check(self.owner.as_ptr()) } == 1; + if is_memoryview || self.owner.as_ref(py).is_instance_of::() { + Err(pyo3::exceptions::PyTypeError::new_err( + "With PyPy, an output of type `bytes` or `memoryview` does not work. See issue pypy/pypy#4918", + )) + } else { + Ok(()) + } + })?; + } Ok(unsafe { std::slice::from_raw_parts_mut(self.buf_ptr() as *mut u8, self.len_bytes()) }) } /// If underlying buffer is c_contiguous pub fn is_c_contiguous(&self) -> bool { - unsafe { ffi::PyBuffer_IsContiguous(&*self.inner as *const ffi::Py_buffer, b'C' as std::os::raw::c_char) != 0 } + unsafe { ffi::PyBuffer_IsContiguous(&*self.inner as *const ffi::Py_buffer, b'C' as std::os::raw::c_char) == 1 } } /// Dimensions for buffer pub fn dimensions(&self) -> usize { @@ -258,6 +271,8 @@ impl<'py> TryFrom<&'py PyAny> for PythonBuffer { let buf = Self { inner: std::pin::Pin::from(buf), pos: 0, + #[cfg(PyPy)] + owner: Python::with_gil(|py| obj.to_object(py)), }; // sanity checks if buf.inner.shape.is_null() { @@ -328,8 +343,9 @@ impl AsBytes for RustyBuffer { fn as_bytes(&self) -> &[u8] { self.inner.get_ref().as_slice() } - fn as_bytes_mut(&mut self) -> &mut [u8] { - self.inner.get_mut().as_mut_slice() + fn as_bytes_mut(&mut self) -> PyResult<&mut [u8]> { + let slice = self.inner.get_mut().as_mut_slice(); + Ok(slice) } } diff --git a/cramjam-python/src/lib.rs b/cramjam-python/src/lib.rs index c494db86..f5fea3ba 100644 --- a/cramjam-python/src/lib.rs +++ b/cramjam-python/src/lib.rs @@ -101,18 +101,18 @@ impl<'a> AsBytes for BytesType<'a> { } } } - fn as_bytes_mut(&mut self) -> &mut [u8] { + fn as_bytes_mut(&mut self) -> PyResult<&mut [u8]> { match self { BytesType::RustyBuffer(b) => { let mut py_ref = b.borrow_mut(); - let bytes = py_ref.as_bytes_mut(); - unsafe { std::slice::from_raw_parts_mut(bytes.as_mut_ptr(), bytes.len()) } + let bytes = py_ref.as_bytes_mut()?; + Ok(unsafe { std::slice::from_raw_parts_mut(bytes.as_mut_ptr(), bytes.len()) }) } - BytesType::PyBuffer(b) => b.as_slice_mut().unwrap(), + BytesType::PyBuffer(b) => b.as_slice_mut(), BytesType::RustyFile(b) => { let mut py_ref = b.borrow_mut(); - let bytes = py_ref.as_bytes_mut(); - unsafe { std::slice::from_raw_parts_mut(bytes.as_mut_ptr(), bytes.len()) } + let bytes = py_ref.as_bytes_mut()?; + Ok(unsafe { std::slice::from_raw_parts_mut(bytes.as_mut_ptr(), bytes.len()) }) } } } @@ -212,7 +212,7 @@ macro_rules! generic { }) }, _ => { - let bytes_out = $output.as_bytes_mut(); + let bytes_out = $output.as_bytes_mut()?; $py.allow_threads(|| { $op(f_in, &mut Cursor::new(bytes_out) $(, $level)? ) }) @@ -237,7 +237,7 @@ macro_rules! generic { }) }, _ => { - let bytes_out = $output.as_bytes_mut(); + let bytes_out = $output.as_bytes_mut()?; $py.allow_threads(|| { $op(bytes_in, &mut Cursor::new(bytes_out) $(, $level)?) }) diff --git a/cramjam-python/src/lz4.rs b/cramjam-python/src/lz4.rs index 016c4470..c7894a00 100644 --- a/cramjam-python/src/lz4.rs +++ b/cramjam-python/src/lz4.rs @@ -146,7 +146,7 @@ pub fn compress_block( #[pyfunction] pub fn decompress_block_into(py: Python, input: BytesType, mut output: BytesType) -> PyResult { let bytes = input.as_bytes(); - let out_bytes = output.as_bytes_mut(); + let out_bytes = output.as_bytes_mut()?; py.allow_threads(|| libcramjam::lz4::block::decompress_into(bytes, out_bytes, Some(true))) .map_err(DecompressionError::from_err) .map(|v| v as _) @@ -180,7 +180,7 @@ pub fn compress_block_into( store_size: Option, ) -> PyResult { let bytes = data.as_bytes(); - let out_bytes = output.as_bytes_mut(); + let out_bytes = output.as_bytes_mut()?; py.allow_threads(|| { libcramjam::lz4::block::compress_into(bytes, out_bytes, compression.map(|v| v as _), acceleration, store_size) }) diff --git a/cramjam-python/src/snappy.rs b/cramjam-python/src/snappy.rs index 1cd9ed83..9133c7af 100644 --- a/cramjam-python/src/snappy.rs +++ b/cramjam-python/src/snappy.rs @@ -100,7 +100,7 @@ pub fn decompress_into(py: Python, input: BytesType, mut output: BytesType) -> P #[pyfunction] pub fn compress_raw_into(py: Python, input: BytesType, mut output: BytesType) -> PyResult { let bytes_in = input.as_bytes(); - let bytes_out = output.as_bytes_mut(); + let bytes_out = output.as_bytes_mut()?; py.allow_threads(|| libcramjam::snappy::raw::compress(bytes_in, bytes_out)) .map_err(CompressionError::from_err) } @@ -109,7 +109,7 @@ pub fn compress_raw_into(py: Python, input: BytesType, mut output: BytesType) -> #[pyfunction] pub fn decompress_raw_into(py: Python, input: BytesType, mut output: BytesType) -> PyResult { let bytes_in = input.as_bytes(); - let bytes_out = output.as_bytes_mut(); + let bytes_out = output.as_bytes_mut()?; py.allow_threads(|| libcramjam::snappy::raw::decompress(bytes_in, bytes_out)) .map_err(DecompressionError::from_err) } diff --git a/cramjam-python/tests/conftest.py b/cramjam-python/tests/conftest.py new file mode 100644 index 00000000..6de6751f --- /dev/null +++ b/cramjam-python/tests/conftest.py @@ -0,0 +1,8 @@ +import platform +import pytest + + +@pytest.fixture(scope='session') +def is_pypy(): + impl = platform.python_implementation() + return impl.lower() == 'pypy' diff --git a/cramjam-python/tests/test_rust_io.py b/cramjam-python/tests/test_rust_io.py index c8f695be..6737262e 100644 --- a/cramjam-python/tests/test_rust_io.py +++ b/cramjam-python/tests/test_rust_io.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize("Obj", (File, Buffer)) -def test_obj_api(tmpdir, Obj): +def test_obj_api(tmpdir, Obj, is_pypy): if isinstance(Obj, File): buf = File(str(tmpdir.join("file.txt"))) else: @@ -30,6 +30,11 @@ def test_obj_api(tmpdir, Obj): ): buf.seek(0) + if isinstance(out, bytes) and is_pypy: + with pytest.raises(OSError): + buf.readinto(out) + continue + expected = b"bytes" buf.readinto(out) diff --git a/cramjam-python/tests/test_variants.py b/cramjam-python/tests/test_variants.py index ecc9181b..1f5713c5 100644 --- a/cramjam-python/tests/test_variants.py +++ b/cramjam-python/tests/test_variants.py @@ -40,7 +40,7 @@ def test_has_version(): @pytest.mark.parametrize("variant_str", VARIANTS) @given(arr=st_np.arrays(st_np.scalar_dtypes(), shape=st.integers(0, int(1e4)))) -def test_variants_different_dtypes(variant_str, arr): +def test_variants_different_dtypes(variant_str, arr, is_pypy): variant = getattr(cramjam, variant_str) compressed = variant.compress(arr) decompressed = variant.decompress(compressed) @@ -49,7 +49,14 @@ def test_variants_different_dtypes(variant_str, arr): # And compress n dims > 1 if arr.shape[0] % 2 == 0: arr = arr.reshape((2, -1)) - compressed = variant.compress(arr) + + if is_pypy: + try: + compressed = variant.compress(arr) + except: + pytest.xfail(reason="PyPy struggles w/ multidim buffer views depending on dtype ie datetime[64]") + else: + compressed = variant.compress(arr) decompressed = variant.decompress(compressed) assert same_same(bytes(decompressed), arr.tobytes()) @@ -89,7 +96,7 @@ def test_variants_raise_exception(variant_str): @pytest.mark.parametrize("variant_str", VARIANTS) @given(raw_data=st.binary()) def test_variants_compress_into( - variant_str, input_type, output_type, raw_data, tmp_path_factory + variant_str, input_type, output_type, raw_data, tmp_path_factory, is_pypy ): variant = getattr(cramjam, variant_str) @@ -124,6 +131,11 @@ def test_variants_compress_into( else: output = output_type(b"0" * compressed_len) + if is_pypy and isinstance(output, (bytes, memoryview)): + with pytest.raises(TypeError): + variant.compress_into(input, output) + return + n_bytes = variant.compress_into(input, output) assert n_bytes == compressed_len @@ -146,7 +158,7 @@ def test_variants_compress_into( @pytest.mark.parametrize("variant_str", VARIANTS) @given(raw_data=st.binary()) def test_variants_decompress_into( - variant_str, input_type, output_type, tmp_path_factory, raw_data + variant_str, input_type, output_type, tmp_path_factory, raw_data, is_pypy ): variant = getattr(cramjam, variant_str) @@ -180,6 +192,11 @@ def test_variants_decompress_into( else: output = output_type(b"0" * len(raw_data)) + if is_pypy and isinstance(output, (bytes, memoryview)): + with pytest.raises(TypeError): + variant.decompress_into(input, output) + return + n_bytes = variant.decompress_into(input, output) assert n_bytes == len(raw_data)