Merge pull request #130 from pycompression/release_1.1.0

Release 1.1.0

.github/workflows/ci.yml

@@ -61,6 +61,7 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
+          - "3.11-dev"
           - "pypy-3.7"
           - "pypy-3.8"
           - "pypy-3.9"

CHANGELOG.rst

@@ -7,6 +7,10 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.1.0
+-----------------
++ Added tests and support for Python 3.11.
+
 version 1.0.1
 ------------------
 + Fixed failing tests and wheel builds for PyPy.

setup.py

@@ -161,7 +161,7 @@ def build_isa_l(compiler_command: str, compiler_options: str):
 
 setup(
     name="isal",
-    version="1.0.1",
+    version="1.1.0",
     description="Faster zlib and gzip compatible compression and "
                 "decompression by providing python bindings for the ISA-L "
                 "library.",
@@ -188,6 +188,7 @@ def build_isa_l(compiler_command: str, compiler_options: str):
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Programming Language :: C",

src/isal/__init__.py

@@ -27,4 +27,4 @@
     "__version__"
 ]
 
-__version__ = "1.0.1"
+__version__ = "1.1.0"

src/isal/_isalmodule.c

@@ -1,12 +1,15 @@
-//  Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-// 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
-// Python Software Foundation; All Rights Reserved
+/*
+Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
+Python Software Foundation; All Rights Reserved
 
-// This file is part of python-isal which is distributed under the 
-// PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
+This file is part of python-isal which is distributed under the 
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
 
-// This file is not originally from the CPython distribution. But it does contain mostly example code
-// from the Python docs. Also dual licensing just for this one file seemed silly.
+This file is not originally from the CPython distribution. But it does 
+contain mostly example code from the Python docs. Also dual licensing just 
+for this one file seemed silly.
+*/
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
@@ -24,20 +27,19 @@ static struct PyModuleDef _isal_module = {
 PyMODINIT_FUNC
 PyInit__isal(void)
 {
-    PyObject *m;
-
-    m = PyModule_Create(&_isal_module);
-    if (m == NULL)
+    PyObject *m = PyModule_Create(&_isal_module);
+    if (m == NULL) {
         return NULL;
-
+    }
     PyModule_AddIntMacro(m, ISAL_MAJOR_VERSION);
     PyModule_AddIntMacro(m, ISAL_MINOR_VERSION);
     PyModule_AddIntMacro(m, ISAL_PATCH_VERSION);
 
     PyObject *isal_version = PyUnicode_FromFormat(
         "%d.%d.%d", ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION);  
-    if (isal_version == NULL)
+    if (isal_version == NULL) {
         return NULL;
+    }
     PyModule_AddObject(m, "ISAL_VERSION", isal_version);
     return m;
 }

src/isal/igzip.py

@@ -167,7 +167,7 @@ def __init__(self, filename=None, mode=None,
                                                   0)
         if self.mode == READ:
             raw = _IGzipReader(self.fileobj)
-            self._buffer = io.BufferedReader(raw, buffer_size=READ_BUFFER_SIZE)
+            self._buffer = io.BufferedReader(raw)
 
     def __repr__(self):
         s = repr(self.fileobj)

src/isal/igzip_libmodule.c

@@ -1,45 +1,47 @@
-//  Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-// 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
-// Python Software Foundation; All Rights Reserved
-
-// This file is part of python-isal which is distributed under the 
-// PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
-
-// This file was modified from Cpython Modules/bz2module.c file from the 3.9
-// branch.
-
-// Changes compared to CPython:
-// - The BZ2Decompressor has been used as a basis for IgzipDecompressor.
-//   Functionality is almost the same. IgzipDecompressor does have a more
-//   elaborate __init__ to set settings. It also implements decompress_buf more
-//   akin to how decompression is implemented in isal_shared.h
-// - Constants were added that are particular to igzip_lib.
-// - Argument parsers were written using th CPython API rather than argument
-//   clinic.
+/* 
+Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
+Python Software Foundation; All Rights Reserved
+
+This file is part of python-isal which is distributed under the 
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
+
+This file was modified from Cpython Modules/bz2module.c file from the 3.9
+branch.
+
+Changes compared to CPython:
+- The BZ2Decompressor has been used as a basis for IgzipDecompressor.
+  Functionality is almost the same. IgzipDecompressor does have a more
+  elaborate __init__ to set settings. It also implements decompress_buf more
+  akin to how decompression is implemented in isal_shared.h
+- Constants were added that are particular to igzip_lib.
+- Argument parsers were written using th CPython API rather than argument
+  clinic.
+*/
 
 #include "isal_shared.h"
 
 typedef struct {
     PyObject_HEAD
-    struct inflate_state state;
-    char eof;           /* T_BOOL expects a char */
     PyObject *unused_data;
     PyObject *zdict;
-    char needs_input;
     uint8_t *input_buffer;
     Py_ssize_t input_buffer_size;
-
     /* inflate_state>avail_in is only 32 bit, so we store the true length
        separately. Conversion and looping is encapsulated in
        decompress_buf() */
     Py_ssize_t avail_in_real;
+    char eof;           /* T_BOOL expects a char */
+    char needs_input;
+    /* Struct inflate state contains a massive buffer at the end. Put it at
+       the end of the IgzipDecompressor so members can be accessed easily. */
+    struct inflate_state state;
 } IgzipDecompressor;
 
 static void
 IgzipDecompressor_dealloc(IgzipDecompressor *self)
 {
-    if(self->input_buffer != NULL)
-        PyMem_Free(self->input_buffer);
+    PyMem_Free(self->input_buffer);
     Py_CLEAR(self->unused_data);
     Py_CLEAR(self->zdict);
     Py_TYPE(self)->tp_free((PyObject *)self);
@@ -55,23 +57,23 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length)
     /* data_size is strictly positive, but because we repeatedly have to
        compare against max_length and PyBytes_GET_SIZE we declare it as
        signed */
-    PyObject * RetVal = NULL;
+    PyObject *RetVal = NULL;
     Py_ssize_t hard_limit;
 
     Py_ssize_t obuflen;
 
     int err;
 
-    // In Python 3.10 sometimes sys.maxsize is passed by default. In those cases
-    // we do want to use DEF_BUF_SIZE as start buffer.
+    /* In Python 3.10 sometimes sys.maxsize is passed by default. In those cases
+       we do want to use DEF_BUF_SIZE as start buffer. */
     if ((max_length < 0) || max_length == PY_SSIZE_T_MAX) {
         hard_limit = PY_SSIZE_T_MAX;
         obuflen = DEF_BUF_SIZE;
     } else {
-        // Assume that decompressor is used in file decompression with a fixed
-        // block size of max_length. In that case we will reach max_length almost
-        // always (except at the end of the file). So it makes sense to allocate
-        // max_length.
+        /* Assume that decompressor is used in file decompression with a fixed
+           block size of max_length. In that case we will reach max_length almost
+           always (except at the end of the file). So it makes sense to allocate
+           max_length. */
         hard_limit = max_length;
         obuflen = max_length;
         if (obuflen > DEF_MAX_INITIAL_BUF_SIZE){
@@ -102,8 +104,10 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length)
                 isal_inflate_error(err);
                 goto error;
             }
-        } while (self->state.avail_out == 0 && self->state.block_state != ISAL_BLOCK_FINISH);
-    } while(self->avail_in_real != 0 && self->state.block_state != ISAL_BLOCK_FINISH);
+        } while (self->state.avail_out == 0 && 
+                 self->state.block_state != ISAL_BLOCK_FINISH);
+    } while(self->avail_in_real != 0 && 
+            self->state.block_state != ISAL_BLOCK_FINISH);
 
     if (self->state.block_state == ISAL_BLOCK_FINISH)
         self->eof = 1;
@@ -183,13 +187,14 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le
         self->needs_input = 0;
         Py_ssize_t bytes_in_bitbuffer = bitbuffer_size(&(self->state));
         if (self->avail_in_real + bytes_in_bitbuffer > 0) {
-            PyObject * new_data = PyBytes_FromStringAndSize(
+            PyObject *new_data = PyBytes_FromStringAndSize(
                 NULL, self->avail_in_real + bytes_in_bitbuffer);
             if (new_data == NULL)
                 goto error;
-            char * new_data_ptr = PyBytes_AS_STRING(new_data);
+            char *new_data_ptr = PyBytes_AS_STRING(new_data);
             bitbuffer_copy(&(self->state), new_data_ptr, bytes_in_bitbuffer);
-            memcpy(new_data_ptr + bytes_in_bitbuffer, self->state.next_in, self->avail_in_real);
+            memcpy(new_data_ptr + bytes_in_bitbuffer, self->state.next_in, 
+                   self->avail_in_real);
             Py_XSETREF(self->unused_data, new_data);
         }
     }
@@ -208,7 +213,7 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le
             /* Discard buffer if it's too small
                (resizing it may needlessly copy the current contents) */
             if (self->input_buffer != NULL &&
-                self->input_buffer_size < self->avail_in_real) {
+                    self->input_buffer_size < self->avail_in_real) {
                 PyMem_Free(self->input_buffer);
                 self->input_buffer = NULL;
             }
@@ -257,13 +262,14 @@ PyDoc_STRVAR(igzip_lib_compress__doc__,
 "    the header and trailer are controlled by the flag parameter.");
 
 #define IGZIP_LIB_COMPRESS_METHODDEF    \
-    {"compress", (PyCFunction)(void(*)(void))igzip_lib_compress, METH_VARARGS|METH_KEYWORDS, igzip_lib_compress__doc__}
+    {"compress", (PyCFunction)(void(*)(void))igzip_lib_compress, \
+     METH_VARARGS|METH_KEYWORDS, igzip_lib_compress__doc__}
 
 static PyObject *
 igzip_lib_compress(PyObject *module, PyObject *args, PyObject *kwargs)
 {
-    char *keywords[] = {"", "level", "flag", "mem_level", "hist_bits", NULL};
-    char *format ="y*|iiii:compress";
+    static char *keywords[] = {"", "level", "flag", "mem_level", "hist_bits", NULL};
+    static char *format ="y*|iiii:compress";
     Py_buffer data = {NULL, NULL};
     int level = ISAL_DEFAULT_COMPRESSION;
     int flag = COMP_DEFLATE;
@@ -298,13 +304,14 @@ PyDoc_STRVAR(igzip_lib_decompress__doc__,
 "    The initial output buffer size.");
 
 #define IGZIP_LIB_DECOMPRESS_METHODDEF    \
-    {"decompress", (PyCFunction)(void(*)(void))igzip_lib_decompress, METH_VARARGS|METH_KEYWORDS, igzip_lib_decompress__doc__}
+    {"decompress", (PyCFunction)(void(*)(void))igzip_lib_decompress, \
+     METH_VARARGS|METH_KEYWORDS, igzip_lib_decompress__doc__}
 
 static PyObject *
 igzip_lib_decompress(PyObject *module, PyObject *args, PyObject *kwargs)
 {
-    char *keywords[] = {"", "flag", "hist_bits", "bufsize", NULL};
-    char *format ="y*|iin:decompress";
+    static char *keywords[] = {"", "flag", "hist_bits", "bufsize", NULL};
+    static char *format ="y*|iin:decompress";
     Py_buffer data = {NULL, NULL};
     int flag = DECOMP_DEFLATE;
     int hist_bits = ISAL_DEF_MAX_HIST_BITS;
@@ -315,7 +322,7 @@ igzip_lib_decompress(PyObject *module, PyObject *args, PyObject *kwargs)
             &data, &flag, &hist_bits, &bufsize)) {
         return NULL;
     }
-    PyObject * return_value = igzip_lib_decompress_impl(&data, flag, hist_bits, bufsize);
+    PyObject *return_value = igzip_lib_decompress_impl(&data, flag, hist_bits, bufsize);
     PyBuffer_Release(&data);
     return return_value;
 }
@@ -340,13 +347,16 @@ PyDoc_STRVAR(igzip_lib_IgzipDecompressor_decompress__doc__,
 "the unused_data attribute.");
 
 #define IGZIP_LIB_IGZIPDECOMPRESSOR_DECOMPRESS_METHODDEF    \
-    {"decompress", (PyCFunction)(void(*)(void))igzip_lib_IgzipDecompressor_decompress, METH_VARARGS|METH_KEYWORDS, igzip_lib_IgzipDecompressor_decompress__doc__}
+    {"decompress", (PyCFunction)(void(*)(void))igzip_lib_IgzipDecompressor_decompress, \
+     METH_VARARGS|METH_KEYWORDS, igzip_lib_IgzipDecompressor_decompress__doc__}
 
 static PyObject *
-igzip_lib_IgzipDecompressor_decompress(IgzipDecompressor *self, PyObject *args, PyObject *kwargs)
+igzip_lib_IgzipDecompressor_decompress(IgzipDecompressor *self, 
+                                       PyObject *args, 
+                                       PyObject *kwargs)
 {
-    char *keywords[] = {"", "max_length", NULL};
-    char *format = "y*|n:decompress";
+    static char *keywords[] = {"", "max_length", NULL};
+    static char *format = "y*|n:decompress";
     Py_buffer data = {NULL, NULL};
     Py_ssize_t max_length = -1;
 
@@ -383,8 +393,8 @@ igzip_lib_IgzipDecompressor__new__(PyTypeObject *type,
                                    PyObject *args, 
                                    PyObject *kwargs)
 {
-    char *keywords[] = {"flag", "hist_bits", "zdict", NULL};
-    char *format = "|iiO:IgzipDecompressor";
+    static char *keywords[] = {"flag", "hist_bits", "zdict", NULL};
+    static char *format = "|iiO:IgzipDecompressor";
     int flag = ISAL_DEFLATE;
     int hist_bits = ISAL_DEF_MAX_HIST_BITS;
     PyObject *zdict = NULL;
@@ -458,8 +468,9 @@ static PyMemberDef IgzipDecompressor_members[] = {
      READONLY, IgzipDecompressor_unused_data__doc__},
     {"needs_input", T_BOOL, offsetof(IgzipDecompressor, needs_input), READONLY,
      IgzipDecompressor_needs_input_doc},
-    {"crc", T_UINT, offsetof(IgzipDecompressor, state) + offsetof(struct inflate_state, crc), READONLY,
-     IgzipDecompressor_crc_doc},
+    {"crc", T_UINT, 
+     offsetof(IgzipDecompressor, state) + offsetof(struct inflate_state, crc), 
+     READONLY, IgzipDecompressor_crc_doc},
     {NULL}
 };
 
@@ -561,9 +572,7 @@ static struct PyModuleDef igzip_lib_module = {
 PyMODINIT_FUNC
 PyInit_igzip_lib(void)
 {
-    PyObject *m;
-
-    m = PyModule_Create(&igzip_lib_module);
+    PyObject *m = PyModule_Create(&igzip_lib_module);
     if (m == NULL)
         return NULL;
 
@@ -580,10 +589,12 @@ PyInit_igzip_lib(void)
         return NULL;
     }
 
-    if (PyType_Ready(&IgzipDecompressor_Type) != 0)
+    if (PyType_Ready(&IgzipDecompressor_Type) != 0) {
         return NULL;
+    }
     Py_INCREF(&IgzipDecompressor_Type);
-    if (PyModule_AddObject(m, "IgzipDecompressor",  (PyObject *)&IgzipDecompressor_Type) < 0) {
+    if (PyModule_AddObject(m, "IgzipDecompressor",  
+                           (PyObject *)&IgzipDecompressor_Type) < 0) {
         return NULL;
     }