Merge pull request #90 from pycompression/release_0.11.1

Release 0.11.1

CHANGELOG.rst

@@ -7,6 +7,13 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.11.1
+------------------
++ Fixed an issue which occurred rarely that caused IgzipDecompressor's
+  unused_data to report back incorrectly. This caused checksum errors when
+  reading gzip files. The issue was more likely to trigger in multi-member gzip
+  files.
+
 version 0.11.0
 ------------------
 In this release the ``python -m isal.igzip`` relatively slow decompression rate

setup.py

@@ -170,7 +170,7 @@ def build_isa_l(compiler_command: str, compiler_options: str):
 
 setup(
     name="isal",
-    version="0.11.0",
+    version="0.11.1",
     description="Faster zlib and gzip compatible compression and "
                 "decompression by providing python bindings for the ISA-L "
                 "library.",

src/isal/__init__.py

@@ -39,4 +39,4 @@
     "__version__"
 ]
 
-__version__ = "0.11.0"
+__version__ = "0.11.1"

src/isal/igzip_lib.pyx

@@ -481,9 +481,8 @@ cdef class IgzipDecompressor:
                 return b""
             if self.eof:
                 self.needs_input = False
-                if self.avail_in_real > 0:
-                    new_data = PyBytes_FromStringAndSize(<char *>self.stream.next_in, self.avail_in_real)
-                    self.unused_data = self._view_bitbuffer() + new_data
+                new_data = PyBytes_FromStringAndSize(<char *>self.stream.next_in, self.avail_in_real)
+                self.unused_data = self._view_bitbuffer() + new_data
             elif self.avail_in_real == 0:
                 self.stream.next_in = NULL
                 self.needs_input = True

tests/test_igzip_lib.py

@@ -17,10 +17,11 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import gzip
 import itertools
 import os
 import pickle
+import zlib
 from typing import NamedTuple
 
 from isal import igzip_lib
@@ -43,6 +44,8 @@ class Flag(NamedTuple):
 
 
 DATA = RAW_DATA[:128 * 1024]
+ZLIB_COMPRESSED = zlib.compress(DATA)
+GZIP_COMPRESSED = gzip.compress(DATA)
 
 COMPRESS_LEVELS = list(range(4))
 HIST_BITS = list(range(16))
@@ -223,3 +226,29 @@ def test_failure(self):
         # Make sure there are no internal consistencies
         with pytest.raises(Exception):
             igzd.decompress(self.BAD_DATA * 30)
+
+
+@pytest.mark.parametrize("test_offset", range(5))
+def test_igzip_decompressor_raw_deflate_unused_data_zlib(test_offset):
+    data = zlib.compress(b"bla")
+    no_header = data[2:]
+    trailer = data[-4:]
+    raw_deflate_incomplete_trailer = no_header[:-test_offset]
+    true_unused_data = trailer[:-test_offset]
+    igzd = IgzipDecompressor(flag=DECOMP_DEFLATE)
+    igzd.decompress(raw_deflate_incomplete_trailer)
+    if igzd.eof:
+        assert igzd.unused_data == true_unused_data
+
+
+@pytest.mark.parametrize("test_offset", range(9))
+def test_igzip_decompressor_raw_deflate_unused_data_gzip(test_offset):
+    data = gzip.compress(b"bla")
+    no_header = data[10:]
+    trailer = data[-8:]
+    raw_deflate_incomplete_trailer = no_header[:-test_offset]
+    true_unused_data = trailer[:-test_offset]
+    igzd = IgzipDecompressor(flag=DECOMP_DEFLATE)
+    igzd.decompress(raw_deflate_incomplete_trailer)
+    if igzd.eof:
+        assert igzd.unused_data == true_unused_data

tox.ini

@@ -9,7 +9,7 @@ passenv=
     PYTHON_ISAL_LINK_DYNAMIC
 commands =
     # Create HTML coverage report for humans and xml coverage report for external services.
-    coverage run --source=isal -m py.test tests
+    coverage run --branch --source=isal -m py.test tests
     # Ignore errors during report generation. Pypy does not generate proper coverage reports.
     coverage html -i
     coverage xml -i