Optionally support LZO via workspaces

Cargo.toml

@@ -7,9 +7,17 @@ license-file = "LICENSE"
 description = "Thin Python bindings to de/compression algorithms in Rust"
 readme = "README.md"
 
+members = [
+    ".",
+    "cramjam-lzo"
+]
+default-members = [
+    ".",
+]
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lib]
-crate-type = ["cdylib"]
+crate-type = ["cdylib", "rlib"]
 
 [features]
 default = ["mimallocator", "extension-module"]

Makefile

@@ -33,6 +33,11 @@ dev-install:
 	pip uninstall cramjam -y
 	pip install cramjam --no-index --find-links dist/
 
+lzo-install:
+	maturin build --manifest-path cramjam-lzo/Cargo.toml --release --out dist --no-sdist --interpreter $(shell which python)
+	pip uninstall cramjam-lzo -y
+	pip install cramjam-lzo --no-index --find-links dist/
+
 pypy-build:
 	maturin build -i $(shell which pypy) --release --out dist
 	pypy ./pypy_patch.py

cramjam-lzo/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "cramjam-lzo"
+version = "2.2.0"
+authors = ["milesgranger <[email protected]>"]
+edition = "2018"
+license-file = "LICENSE"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "cramjam_lzo"
+crate-type = ["cdylib"]
+
+[features]
+default = ["extension-module"]
+extension-module = ["pyo3/extension-module"]
+
+[dependencies]
+minilzo3 = "^0.1"
+cramjam = { path = "../", version = "2.2.0" }
+pyo3 = { version = "0.13.2", default-features = false, features = ["macros"] }

cramjam-lzo/README.md

@@ -0,0 +1,19 @@
+An extension to cramjam for LZO
+
+Follows the same API, but is not distributed
+with cramjam due to LZO having [GPL-2.0 license](LICENSE) which is not
+compatible with cramjam's [MIT license](../LICENSE).
+
+Therefore, should you choose to use this, you're then subject to the 
+contraints of GPL-2.0
+
+---
+```bash
+pip install cramjam
+pip install cramjam-lzo
+```
+
+```python
+# LZO will then be a submodule to cramjam
+from cramjam import lzo
+```

cramjam-lzo/src/lib.rs

@@ -0,0 +1,65 @@
+//! lzo de/compression interface
+use cramjam::{CompressionError, DecompressionError};
+use cramjam::{AsBytes, RustyBuffer, to_py_err, BytesType};
+use pyo3::prelude::*;
+use pyo3::wrap_pyfunction;
+use pyo3::PyResult;
+
+/// LZO decompression
+///
+/// Python Example
+/// --------------
+/// ```python
+/// >>> cramjam.lzo.decompress(compressed_raw_bytes)
+/// ```
+#[pyfunction]
+#[allow(unused_variables)]
+pub fn decompress(data: BytesType, output_len: Option<usize>) -> PyResult<RustyBuffer> {
+    let output = to_py_err!(DecompressionError -> minilzo3::decompress_vec(data.as_bytes()))?;
+    Ok(RustyBuffer::from(output))
+}
+
+/// LZO compression
+///
+/// This follows the header format of `python-lzo` where the first byte indicates if it's level 1
+/// compression (default; and only one implemented here thus far) and the next four bytes are
+/// u32 big endian formatted bytes indicating the length of the original input, before compression.
+///
+/// Python Example
+/// --------------
+/// ```python
+/// >>> cramjam.lzo.compress(b'some bytes here')
+/// ```
+#[pyfunction]
+#[allow(unused_variables)]
+pub fn compress(py: Python, data: PyObject, output_len: Option<usize>) -> PyResult<RustyBuffer> {
+    let data: BytesType = data.extract(py)?;
+    let output = to_py_err!(CompressionError -> minilzo3::compress_vec(data.as_bytes(), true))?;
+    Ok(RustyBuffer::from(output))
+}
+
+/// Compress raw format directly into an output buffer
+#[pyfunction]
+pub fn compress_into(input: BytesType, mut output: BytesType) -> PyResult<usize> {
+    let output = minilzo3::compress(input.as_bytes(), output.as_bytes_mut(), true);
+    to_py_err!(CompressionError -> output)
+}
+
+/// Decompress raw format directly into an output buffer
+#[pyfunction]
+pub fn decompress_into(input: BytesType, mut output: BytesType) -> PyResult<usize> {
+    let output = minilzo3::decompress(input.as_bytes(), output.as_bytes_mut());
+    to_py_err!(DecompressionError -> output)
+}
+
+#[pymodule]
+fn cramjam_lzo(_py: Python, m: &PyModule) -> PyResult<()> {
+    m.add("__version__", env!("CARGO_PKG_VERSION"))?;
+    m.add_class::<cramjam::RustyFile>()?;
+    m.add_class::<cramjam::RustyBuffer>()?;
+    m.add_function(wrap_pyfunction!(compress, m)?)?;
+    m.add_function(wrap_pyfunction!(decompress, m)?)?;
+    m.add_function(wrap_pyfunction!(compress_into, m)?)?;
+    m.add_function(wrap_pyfunction!(decompress_into, m)?)?;
+    Ok(())
+}

src/io.rs

@@ -14,8 +14,11 @@ use pyo3::{ffi, PySequenceProtocol};
 use pyo3::{AsPyPointer, PyObjectProtocol};
 use std::path::PathBuf;
 
-pub(crate) trait AsBytes {
+/// Obtain a bytes view of an object's data
+pub trait AsBytes {
+    /// Immutable bytes slice
     fn as_bytes(&self) -> &[u8];
+    /// Mutable bytes slice
     fn as_bytes_mut(&mut self) -> &mut [u8];
 }
 

src/lib.rs

@@ -62,8 +62,8 @@ pub mod zstd;
 
 use pyo3::prelude::*;
 
-use crate::io::{AsBytes, RustyBuffer, RustyFile, RustyNumpyArray, RustyPyByteArray, RustyPyBytes};
-use exceptions::{CompressionError, DecompressionError};
+pub use crate::exceptions::{CompressionError, DecompressionError};
+pub use crate::io::{AsBytes, RustyBuffer, RustyFile, RustyNumpyArray, RustyPyByteArray, RustyPyBytes};
 use std::io::{Read, Seek, SeekFrom, Write};
 
 #[cfg(feature = "mimallocator")]
@@ -172,7 +172,8 @@ impl<'a> Seek for BytesType<'a> {
 }
 
 impl<'a> BytesType<'a> {
-    fn len(&self) -> usize {
+    /// Refers to the length of bytes for the underlying variant.
+    pub fn len(&self) -> usize {
         self.as_bytes().len()
     }
 }
@@ -240,6 +241,9 @@ fn cramjam(py: Python, m: &PyModule) -> PyResult<()> {
     make_submodule!(py -> m -> deflate);
     make_submodule!(py -> m -> zstd);
 
+    if let Ok(lzo) = PyModule::import(py, "cramjam_lzo") {
+        m.add("lzo", lzo)?;
+    }
     Ok(())
 }