Add zlib, izlib and ideflate (#184)

* Add zlib, izlib and ideflate
* Bump minor version -> 2.9.0-rc1

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cramjam-python"
-version = "2.8.5-rc1"
+version = "2.9.0-rc1"
 authors = ["Miles Granger <[email protected]>"]
 edition = "2021"
 license = "MIT"
@@ -14,7 +14,7 @@ name = "cramjam"
 crate-type = ["cdylib"]
 
 [features]
-default                  = ["extension-module", "snappy", "lz4", "bzip2", "brotli", "xz", "zstd", "gzip", "deflate", "blosc2", "igzip"]
+default                  = ["extension-module", "snappy", "lz4", "bzip2", "brotli", "xz", "zstd", "gzip", "zlib", "deflate", "blosc2", "igzip", "ideflate", "izlib"]
 extension-module         = ["pyo3/extension-module"]
 generate-import-lib      = ["pyo3/generate-import-lib"]  # needed for Windows PyPy builds
 
@@ -28,16 +28,27 @@ xz                       = ["xz-static"]
 xz-static                = ["libcramjam/xz-static"]
 xz-shared                = ["libcramjam/xz-shared"]
 
+# ISA-L stuff
 igzip                    = ["igzip-static"]
 igzip-static             = ["libcramjam/igzip-static"]
 igzip-shared             = ["libcramjam/igzip-shared"]
+ideflate                 = ["ideflate-static"]
+ideflate-static          = ["libcramjam/ideflate-static"]
+ideflate-shared          = ["libcramjam/ideflate-shared"]
+izlib                    = ["izlib-static"]
+izlib-static             = ["libcramjam/izlib-static"]
+izlib-shared             = ["libcramjam/izlib-shared"]
 use-system-isal-static   = ["libcramjam/use-system-isal", "libcramjam/igzip-static"]
 use-system-isal-shared   = ["libcramjam/use-system-isal", "libcramjam/igzip-shared"]
 
 gzip                     = ["gzip-static"]
 gzip-static              = ["libcramjam/gzip-static"]
 gzip-shared              = ["libcramjam/gzip-shared"]
 
+zlib                     = ["zlib-static"]
+zlib-static              = ["libcramjam/zlib-static"]
+zlib-shared              = ["libcramjam/zlib-shared"]
+
 deflate                  = ["deflate-static"]
 deflate-static           = ["libcramjam/deflate-static"]
 deflate-shared           = ["libcramjam/deflate-shared"]
@@ -53,7 +64,7 @@ wasm32-compat            = ["libcramjam/wasm32-compat"]
 
 [dependencies]
 pyo3 = { version = "^0.22", default-features = false, features = ["macros"] }
-libcramjam = { version = "^0.5", default-features = false }
+libcramjam = { version = "^0.6", default-features = false }
 
 [build-dependencies]
 pyo3-build-config = "^0.22"

README.md

@@ -41,11 +41,15 @@ Available algorithms:
 - [X] Bzip2          `cramjam.bzip2`
 - [X] Lz4              `cramjam.lz4`
 - [X] Gzip            `cramjam.gzip`
+- [X] Zlib            `cramjam.zlib`
 - [X] Deflate       `cramjam.deflate`
 - [X] ZSTD           `cramjam.zstd`
 - [X] XZ / LZMA  `cramjam.xz`
 - [X] Blosc2        `cramjam.experimental.blosc2`
-- [X] IGzip          `cramjam.experimental.igzip` (only on 64-bit targets)
+- [X] ISA-L backend
+  - [X] igzip          `cramjam.experimental.igzip` (only on 64-bit targets)
+  - [X] ideflate          `cramjam.experimental.ideflate` (only on 64-bit targets)
+  - [X] izlib          `cramjam.experimental.izlib` (only on 64-bit targets)
 
 All available for use as:
 

src/experimental.rs

@@ -13,10 +13,24 @@ pub mod experimental {
     #[pymodule_export]
     use crate::blosc2::blosc2;
 
+    #[cfg(all(
+        any(feature = "ideflate", feature = "ideflate-static", feature = "ideflate-shared"),
+        target_pointer_width = "64"
+    ))]
+    #[pymodule_export]
+    use crate::ideflate::ideflate;
+
     #[cfg(all(
         any(feature = "igzip", feature = "igzip-static", feature = "igzip-shared"),
         target_pointer_width = "64"
     ))]
     #[pymodule_export]
     use crate::igzip::igzip;
+
+    #[cfg(all(
+        any(feature = "izlib", feature = "izlib-static", feature = "izlib-shared"),
+        target_pointer_width = "64"
+    ))]
+    #[pymodule_export]
+    use crate::izlib::izlib;
 }

src/ideflate.rs

@@ -0,0 +1,107 @@
+//! ideflate de/compression interface
+use pyo3::prelude::*;
+
+/// ideflate de/compression interface
+#[pymodule]
+pub mod ideflate {
+
+    use crate::exceptions::{CompressionError, DecompressionError};
+    use crate::io::{AsBytes, RustyBuffer};
+    use crate::BytesType;
+    use pyo3::prelude::*;
+    use pyo3::PyResult;
+    use std::io::Cursor;
+
+    const DEFAULT_COMPRESSION_LEVEL: u32 = 6;
+
+    /// ideflate decompression.
+    ///
+    /// Python Example
+    /// --------------
+    /// ```python
+    /// >>> cramjam.gzip.decompress(compressed_bytes, output_len=Optional[int])
+    /// ```
+    #[pyfunction]
+    #[pyo3(signature = (data, output_len=None))]
+    pub fn decompress(py: Python, data: BytesType, output_len: Option<usize>) -> PyResult<RustyBuffer> {
+        crate::generic!(py, libcramjam::ideflate::decompress[data], output_len = output_len)
+            .map_err(DecompressionError::from_err)
+    }
+
+    /// ideflate compression.
+    ///
+    /// Python Example
+    /// --------------
+    /// ```python
+    /// >>> cramjam.gzip.compress(b'some bytes here', level=2, output_len=Optional[int])  # Level defaults to 6
+    /// ```
+    #[pyfunction]
+    #[pyo3(signature = (data, level=None, output_len=None))]
+    pub fn compress(
+        py: Python,
+        data: BytesType,
+        level: Option<u32>,
+        output_len: Option<usize>,
+    ) -> PyResult<RustyBuffer> {
+        crate::generic!(py, libcramjam::ideflate::compress[data], output_len = output_len, level)
+            .map_err(CompressionError::from_err)
+    }
+
+    /// Compress directly into an output buffer
+    #[pyfunction]
+    #[pyo3(signature = (input, output, level=None))]
+    pub fn compress_into(py: Python, input: BytesType, mut output: BytesType, level: Option<u32>) -> PyResult<usize> {
+        crate::generic!(py, libcramjam::ideflate::compress[input, output], level).map_err(CompressionError::from_err)
+    }
+
+    /// Decompress directly into an output buffer
+    #[pyfunction]
+    pub fn decompress_into(py: Python, input: BytesType, mut output: BytesType) -> PyResult<usize> {
+        crate::generic!(py, libcramjam::ideflate::decompress[input, output]).map_err(DecompressionError::from_err)
+    }
+
+    /// ideflate Compressor object for streaming compression
+    #[pyclass(unsendable)] // TODO: make sendable
+    pub struct Compressor {
+        inner: Option<libcramjam::ideflate::isal::write::GzipEncoder<Cursor<Vec<u8>>>>,
+    }
+
+    #[pymethods]
+    impl Compressor {
+        /// Initialize a new `Compressor` instance.
+        #[new]
+        #[pyo3(signature = (level=None))]
+        pub fn __init__(level: Option<u32>) -> PyResult<Self> {
+            let level = level.unwrap_or(DEFAULT_COMPRESSION_LEVEL);
+            let inner = libcramjam::ideflate::isal::write::GzipEncoder::new(
+                Cursor::new(vec![]),
+                libcramjam::ideflate::isal::CompressionLevel::try_from(level as isize)
+                    .map_err(CompressionError::from_err)?,
+            );
+            Ok(Self { inner: Some(inner) })
+        }
+
+        /// Compress input into the current compressor's stream.
+        pub fn compress(&mut self, input: &[u8]) -> PyResult<usize> {
+            crate::io::stream_compress(&mut self.inner, input)
+        }
+
+        /// Flush and return current compressed stream
+        pub fn flush(&mut self) -> PyResult<RustyBuffer> {
+            crate::io::stream_flush(&mut self.inner, |e| e.get_ref_mut())
+        }
+
+        /// Consume the current compressor state and return the compressed stream
+        /// **NB** The compressor will not be usable after this method is called.
+        pub fn finish(&mut self) -> PyResult<RustyBuffer> {
+            crate::io::stream_finish(&mut self.inner, |inner| inner.finish().map(|c| c.into_inner()))
+        }
+    }
+
+    mod _decompressor {
+        use super::*;
+        crate::make_decompressor!(ideflate);
+    }
+    #[pymodule_export]
+    use _decompressor::Decompressor;
+}

src/izlib.rs

@@ -0,0 +1,107 @@
+//! izlib de/compression interface
+use pyo3::prelude::*;
+
+/// izlib de/compression interface
+#[pymodule]
+pub mod izlib {
+
+    use crate::exceptions::{CompressionError, DecompressionError};
+    use crate::io::{AsBytes, RustyBuffer};
+    use crate::BytesType;
+    use pyo3::prelude::*;
+    use pyo3::PyResult;
+    use std::io::Cursor;
+
+    const DEFAULT_COMPRESSION_LEVEL: u32 = 6;
+
+    /// izlib decompression.
+    ///
+    /// Python Example
+    /// --------------
+    /// ```python
+    /// >>> cramjam.gzip.decompress(compressed_bytes, output_len=Optional[int])
+    /// ```
+    #[pyfunction]
+    #[pyo3(signature = (data, output_len=None))]
+    pub fn decompress(py: Python, data: BytesType, output_len: Option<usize>) -> PyResult<RustyBuffer> {
+        crate::generic!(py, libcramjam::izlib::decompress[data], output_len = output_len)
+            .map_err(DecompressionError::from_err)
+    }
+
+    /// izlib compression.
+    ///
+    /// Python Example
+    /// --------------
+    /// ```python
+    /// >>> cramjam.gzip.compress(b'some bytes here', level=2, output_len=Optional[int])  # Level defaults to 6
+    /// ```
+    #[pyfunction]
+    #[pyo3(signature = (data, level=None, output_len=None))]
+    pub fn compress(
+        py: Python,
+        data: BytesType,
+        level: Option<u32>,
+        output_len: Option<usize>,
+    ) -> PyResult<RustyBuffer> {
+        crate::generic!(py, libcramjam::izlib::compress[data], output_len = output_len, level)
+            .map_err(CompressionError::from_err)
+    }
+
+    /// Compress directly into an output buffer
+    #[pyfunction]
+    #[pyo3(signature = (input, output, level=None))]
+    pub fn compress_into(py: Python, input: BytesType, mut output: BytesType, level: Option<u32>) -> PyResult<usize> {
+        crate::generic!(py, libcramjam::izlib::compress[input, output], level).map_err(CompressionError::from_err)
+    }
+
+    /// Decompress directly into an output buffer
+    #[pyfunction]
+    pub fn decompress_into(py: Python, input: BytesType, mut output: BytesType) -> PyResult<usize> {
+        crate::generic!(py, libcramjam::izlib::decompress[input, output]).map_err(DecompressionError::from_err)
+    }
+
+    /// izlib Compressor object for streaming compression
+    #[pyclass(unsendable)] // TODO: make sendable
+    pub struct Compressor {
+        inner: Option<libcramjam::izlib::isal::write::GzipEncoder<Cursor<Vec<u8>>>>,
+    }
+
+    #[pymethods]
+    impl Compressor {
+        /// Initialize a new `Compressor` instance.
+        #[new]
+        #[pyo3(signature = (level=None))]
+        pub fn __init__(level: Option<u32>) -> PyResult<Self> {
+            let level = level.unwrap_or(DEFAULT_COMPRESSION_LEVEL);
+            let inner = libcramjam::izlib::isal::write::GzipEncoder::new(
+                Cursor::new(vec![]),
+                libcramjam::izlib::isal::CompressionLevel::try_from(level as isize)
+                    .map_err(CompressionError::from_err)?,
+            );
+            Ok(Self { inner: Some(inner) })
+        }
+
+        /// Compress input into the current compressor's stream.
+        pub fn compress(&mut self, input: &[u8]) -> PyResult<usize> {
+            crate::io::stream_compress(&mut self.inner, input)
+        }
+
+        /// Flush and return current compressed stream
+        pub fn flush(&mut self) -> PyResult<RustyBuffer> {
+            crate::io::stream_flush(&mut self.inner, |e| e.get_ref_mut())
+        }
+
+        /// Consume the current compressor state and return the compressed stream
+        /// **NB** The compressor will not be usable after this method is called.
+        pub fn finish(&mut self) -> PyResult<RustyBuffer> {
+            crate::io::stream_finish(&mut self.inner, |inner| inner.finish().map(|c| c.into_inner()))
+        }
+    }
+
+    mod _decompressor {
+        use super::*;
+        crate::make_decompressor!(izlib);
+    }
+    #[pymodule_export]
+    use _decompressor::Decompressor;
+}

src/lib.rs

@@ -65,17 +65,29 @@ pub mod bzip2;
 pub mod deflate;
 #[cfg(any(feature = "gzip", feature = "gzip-static", feature = "gzip-shared"))]
 pub mod gzip;
+#[cfg(all(
+    any(feature = "ideflate", feature = "ideflate-static", feature = "ideflate-shared"),
+    target_pointer_width = "64"
+))]
+pub mod ideflate;
 #[cfg(all(
     any(feature = "igzip", feature = "igzip-static", feature = "igzip-shared"),
     target_pointer_width = "64"
 ))]
 pub mod igzip;
+#[cfg(all(
+    any(feature = "izlib", feature = "izlib-static", feature = "izlib-shared"),
+    target_pointer_width = "64"
+))]
+pub mod izlib;
 #[cfg(feature = "lz4")]
 pub mod lz4;
 #[cfg(feature = "snappy")]
 pub mod snappy;
 #[cfg(any(feature = "xz", feature = "xz-static", feature = "xz-shared"))]
 pub mod xz;
+#[cfg(any(feature = "zlib", feature = "zlib-static", feature = "zlib-shared"))]
+pub mod zlib;
 #[cfg(feature = "zstd")]
 pub mod zstd;
 
@@ -416,10 +428,6 @@ mod cramjam {
     #[pymodule_export]
     use crate::brotli::brotli;
 
-    #[cfg(any(feature = "deflate", feature = "deflate-static", feature = "deflate-shared"))]
-    #[pymodule_export]
-    use crate::deflate::deflate;
-
     #[cfg(any(feature = "xz", feature = "xz-static", feature = "xz-shared"))]
     #[pymodule_export]
     use crate::xz::xz;
@@ -432,6 +440,14 @@ mod cramjam {
     #[pymodule_export]
     use crate::gzip::gzip;
 
+    #[cfg(any(feature = "zlib", feature = "zlib-static", feature = "zlib-shared"))]
+    #[pymodule_export]
+    use crate::zlib::zlib;
+
+    #[cfg(any(feature = "deflate", feature = "deflate-static", feature = "deflate-shared"))]
+    #[pymodule_export]
+    use crate::deflate::deflate;
+
     #[pymodule_export]
     use crate::experimental::experimental;
 }