From 8f4bae5514d9c22bd0598c1da6e8e8dabc3c0dde Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 23 Jul 2021 09:48:41 +0200 Subject: [PATCH 1/6] Update version --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3a8510b2..84f90beb 100644 --- a/setup.py +++ b/setup.py @@ -170,7 +170,7 @@ def build_isa_l(compiler_command: str, compiler_options: str): setup( name="isal", - version="0.11.0", + version="1.0.0-dev", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index bfc6bcdf..b06712ca 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -39,4 +39,4 @@ "__version__" ] -__version__ = "0.11.0" +__version__ = "1.0.0-dev" From 1c2dc24e3951bf0475a232889dbe934cc5ec1e62 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 23 Jul 2021 16:06:55 +0200 Subject: [PATCH 2/6] enable branch coverage --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4011b52e..3ae90ba9 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ passenv= PYTHON_ISAL_LINK_DYNAMIC commands = # Create HTML coverage report for humans and xml coverage report for external services. - coverage run --source=isal -m py.test tests + coverage run --branch --source=isal -m py.test tests # Ignore errors during report generation. Pypy does not generate proper coverage reports. coverage html -i coverage xml -i From 9f2e4ac00dd6af997ffb7aadf5f146245de3f041 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Aug 2021 14:07:09 +0200 Subject: [PATCH 3/6] Add tests to ensure unused_data behaves correctly when using a raw_deflate igzipdecompressor on gzip or zlib data. --- tests/test_igzip_lib.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index acce6250..a0034cc8 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -17,10 +17,11 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - +import gzip import itertools import os import pickle +import zlib from typing import NamedTuple from isal import igzip_lib @@ -43,6 +44,8 @@ class Flag(NamedTuple): DATA = RAW_DATA[:128 * 1024] +ZLIB_COMPRESSED = zlib.compress(DATA) +GZIP_COMPRESSED = gzip.compress(DATA) COMPRESS_LEVELS = list(range(4)) HIST_BITS = list(range(16)) @@ -223,3 +226,29 @@ def test_failure(self): # Make sure there are no internal consistencies with pytest.raises(Exception): igzd.decompress(self.BAD_DATA * 30) + + +@pytest.mark.parametrize("test_offset", range(5)) +def test_igzip_decompressor_raw_deflate_unused_data_zlib(test_offset): + data = zlib.compress(b"bla") + no_header = data[2:] + trailer = data[-4:] + raw_deflate_incomplete_trailer = no_header[:-test_offset] + true_unused_data = trailer[:-test_offset] + igzd = IgzipDecompressor(flag=DECOMP_DEFLATE) + igzd.decompress(raw_deflate_incomplete_trailer) + if igzd.eof: + assert igzd.unused_data == true_unused_data + + +@pytest.mark.parametrize("test_offset", range(9)) +def test_igzip_decompressor_raw_deflate_unused_data_gzip(test_offset): + data = gzip.compress(b"bla") + no_header = data[10:] + trailer = data[-8:] + raw_deflate_incomplete_trailer = no_header[:-test_offset] + true_unused_data = trailer[:-test_offset] + igzd = IgzipDecompressor(flag=DECOMP_DEFLATE) + igzd.decompress(raw_deflate_incomplete_trailer) + if igzd.eof: + assert igzd.unused_data == true_unused_data From dcd7db222951720980025c56f878cedd17b679a1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Aug 2021 14:30:02 +0200 Subject: [PATCH 4/6] Unused_data is always generated to ensure bitbuffer is read. --- src/isal/igzip_lib.pyx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/isal/igzip_lib.pyx b/src/isal/igzip_lib.pyx index 0b2e8c42..af2f2069 100644 --- a/src/isal/igzip_lib.pyx +++ b/src/isal/igzip_lib.pyx @@ -481,9 +481,8 @@ cdef class IgzipDecompressor: return b"" if self.eof: self.needs_input = False - if self.avail_in_real > 0: - new_data = PyBytes_FromStringAndSize(self.stream.next_in, self.avail_in_real) - self.unused_data = self._view_bitbuffer() + new_data + new_data = PyBytes_FromStringAndSize(self.stream.next_in, self.avail_in_real) + self.unused_data = self._view_bitbuffer() + new_data elif self.avail_in_real == 0: self.stream.next_in = NULL self.needs_input = True From db2b0a009ccc7d64667bf93eafd4db2d7c20184b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Aug 2021 14:40:08 +0200 Subject: [PATCH 5/6] Update changelog with fix for IgzipDecompressor.unused_data --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 403709fe..1b1e33cd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 0.11.1 +------------------ ++ Fixed an issue which occurred rarely that caused IgzipDecompressor's + unused_data to report back incorrectly. This caused checksum errors when + reading gzip files. The issue was more likely to trigger in multi-member gzip + files. + version 0.11.0 ------------------ In this release the ``python -m isal.igzip`` relatively slow decompression rate From b96148d37692ae8751328ab10d319457af4dc691 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Sat, 28 Aug 2021 12:03:37 +0200 Subject: [PATCH 6/6] Stable version --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 84f90beb..fccde541 100644 --- a/setup.py +++ b/setup.py @@ -170,7 +170,7 @@ def build_isa_l(compiler_command: str, compiler_options: str): setup( name="isal", - version="1.0.0-dev", + version="0.11.1", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index b06712ca..4a1f6717 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -39,4 +39,4 @@ "__version__" ] -__version__ = "1.0.0-dev" +__version__ = "0.11.1"