Add support for Python 3.10

.flake8

@@ -4,6 +4,7 @@ max-line-length = 125
 # Style issues are handled by yapf formatting
 ignore =
   E126,  # continuation line over-indented for hanging indent
+  E501,  # line length handled by yapf
   W503,  # line break before binary operator
   E251,  # unexpected spaces around keyword / parameter equals
   E722,  # do not use bare 'except' (pylint handles this)

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
     steps:
     - uses: actions/checkout@v1
 

.pylintrc

@@ -4,6 +4,7 @@ jobs=4
 [MESSAGES CONTROL]
 disable=
   bad-continuation,
+  duplicate-code,
   fixme,
   import-outside-toplevel,
   invalid-name,

karapace/config.py

@@ -101,7 +101,7 @@ def read_config(config_handler: IO) -> Config:
         config = set_config_defaults(config)
         return config
     except Exception as ex:
-        raise InvalidConfiguration(ex)
+        raise InvalidConfiguration(ex)  # pylint: disable=raise-missing-from
 
 
 def create_ssl_context(config: Config) -> ssl.SSLContext:
@@ -127,7 +127,6 @@ def create_ssl_context(config: Config) -> ssl.SSLContext:
         if not hasattr(ssl, 'VERIFY_CRL_CHECK_LEAF'):
             raise RuntimeError('This version of Python does not support ssl_crlfile!')
         ssl_context.load_verify_locations(config['ssl_crlfile'])
-        # pylint: disable=no-member
         ssl_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF
     if config.get('ssl_ciphers'):
         ssl_context.set_ciphers(config['ssl_ciphers'])

karapace/kafka_rest_apis/__init__.py

@@ -400,7 +400,7 @@ async def topic_publish(self, topic: str, content_type: str, *, request):
         await self.publish(topic, None, content_type, request.content_type, request.json)
 
     @staticmethod
-    def validate_partition_id(partition_id: str, content_type: str) -> int:
+    def validate_partition_id(partition_id: str, content_type: str) -> int:  # pylint: disable=inconsistent-return-statements
         try:
             return int(partition_id)
         except ValueError:

karapace/karapace.py

@@ -23,7 +23,6 @@ def __init__(self, config: dict) -> None:
         sentry_config = config.get("sentry", {"dsn": None}).copy()
         super().__init__(app_name="karapace", sentry_config=sentry_config)
 
-        self.config = {}
         self.producer = None
         self.kafka_timeout = 10
         self.config = config

karapace/master_coordinator.py

@@ -113,10 +113,12 @@ def _on_join_complete(self, generation, member_id, protocol, member_assignment_b
         else:
             self.master_url = master_url
             self.are_we_master = False
+        # pylint: disable=super-with-arguments
         return super(SchemaCoordinator, self)._on_join_complete(generation, member_id, protocol, member_assignment_bytes)
 
     def _on_join_follower(self):
         self.log.info("We are a follower, not a master")
+        # pylint: disable=super-with-arguments
         return super(SchemaCoordinator, self)._on_join_follower()
 
 

karapace/protobuf/encoding_variants.py

@@ -35,6 +35,7 @@ def read_indexes(bio: BytesIO) -> List[int]:
         size: int = read_varint(bio)
     except EOFError:
         # TODO: change exception
+        # pylint: disable=raise-missing-from
         raise IllegalArgumentException("problem with reading binary data")
     if size == 0:
         return [0]

karapace/protobuf/io.py

@@ -35,6 +35,7 @@ def find_message_name(schema: ProtobufSchema, indexes: List[int]) -> str:
         try:
             message = types[index]
         except IndexError:
+            # pylint: disable=raise-missing-from
             raise IllegalArgumentException(f"Invalid message indexes: {indexes}")
 
         if message and isinstance(message, MessageElement):
@@ -129,6 +130,7 @@ def write(self, datum: dict, writer: BytesIO) -> None:
         try:
             dict_to_protobuf(class_instance, datum)
         except Exception:
+            # pylint: disable=raise-missing-from
             raise ProtobufTypeException(self._writer_schema, datum)
 
         writer.write(class_instance.SerializeToString())

karapace/protobuf/protobuf_to_dict.py

@@ -199,6 +199,7 @@ def _get_field_mapping(pb, dict_value, strict):
         try:
             ext_num = int(ext_num)
         except ValueError:
+            # pylint: disable=raise-missing-from
             raise ValueError("Extension keys must be integers.")
         # pylint: disable=protected-access
         if ext_num not in pb._extensions_by_number:
@@ -295,6 +296,7 @@ def _string_to_enum(field, input_value, strict=False):
         input_value = field.enum_type.values_by_name[input_value].number
     except KeyError:
         if strict:
+            # pylint: disable=raise-missing-from
             raise KeyError("`%s` is not a valid value for field `%s`" % (input_value, field.name))
         return _string_to_enum(field, input_value.upper(), strict=True)
     return input_value
@@ -328,7 +330,7 @@ def validate_dict_for_required_pb_fields(pb, dic):
     Take a look at the tests for an example.
     """
     missing_fields = []
-    for field, field_name, field_options in get_field_names_and_options(pb):
+    for _field, field_name, field_options in get_field_names_and_options(pb):
         if not field_options.get('is_optional', False) and field_name not in dic:
             missing_fields.append(field_name)
     if missing_fields:

karapace/protobuf/syntax_reader.py

@@ -173,7 +173,7 @@ def read_word(self) -> str:
         self.expect(start < self.pos, "expected a word")
         return self.data[start:self.pos]
 
-    def read_int(self) -> int:
+    def read_int(self) -> int:  # pylint: disable=inconsistent-return-statements
         """ Reads an integer and returns it. """
         tag: str = self.read_word()
         try:

karapace/protobuf/utils.py

@@ -45,7 +45,7 @@ def try_to_schema(obj: 'OptionElement') -> str:
     except AttributeError:
         if isinstance(obj, str):
             return obj
-        raise AttributeError
+        raise
 
 
 def append_indented(data: List[str], value: str) -> None:

karapace/rapu.py

@@ -83,11 +83,11 @@ def __init__(
         self.json = None
 
     @overload
-    def get_header(self, header: str) -> Optional[str]:  # pylint: disable=no-self-use
+    def get_header(self, header: str) -> Optional[str]:
         ...
 
     @overload
-    def get_header(self, header: str, default_value: str) -> str:  # pylint: disable=no-self-use
+    def get_header(self, header: str, default_value: str) -> str:
         ...
 
     def get_header(self, header, default_value=None):
@@ -286,11 +286,11 @@ async def _handle_request(
                     body_string = body.decode(charset)
                     rapu_request.json = jsonlib.loads(body_string)
                 except jsonlib.decoder.JSONDecodeError:
-                    raise HTTPResponse(body="Invalid request JSON body", status=HTTPStatus.BAD_REQUEST)
+                    raise HTTPResponse(body="Invalid request JSON body", status=HTTPStatus.BAD_REQUEST)  # pylint: disable=raise-missing-from
                 except UnicodeDecodeError:
-                    raise HTTPResponse(body=f"Request body is not valid {charset}", status=HTTPStatus.BAD_REQUEST)
+                    raise HTTPResponse(body=f"Request body is not valid {charset}", status=HTTPStatus.BAD_REQUEST)  # pylint: disable=raise-missing-from
                 except LookupError:
-                    raise HTTPResponse(body=f"Unknown charset {charset}", status=HTTPStatus.BAD_REQUEST)
+                    raise HTTPResponse(body=f"Unknown charset {charset}", status=HTTPStatus.BAD_REQUEST)  # pylint: disable=raise-missing-from
             else:
                 if body not in {b"", b"{}"}:
                     raise HTTPResponse(body="No request body allowed for this operation", status=HTTPStatus.BAD_REQUEST)

karapace/schema_reader.py

@@ -80,7 +80,7 @@ def parse_json(schema_str: str):
             raise InvalidSchema from e
 
     @staticmethod
-    def parse_avro(schema_str: str):  # pylint: disable=inconsistent-return-statements
+    def parse_avro(schema_str: str):
         try:
             ts = TypedSchema(parse_avro_schema_definition(schema_str), SchemaType.AVRO, schema_str)
             return ts

karapace/serialization.py

@@ -6,7 +6,7 @@
 from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter
 from karapace.schema_reader import InvalidSchema, SchemaType, TypedSchema
 from karapace.utils import Client, json_encode
-from typing import Dict, Optional
+from typing import Dict, Optional, Tuple
 from urllib.parse import quote
 
 import asyncio
@@ -82,7 +82,7 @@ async def post_new_schema(self, subject: str, schema: TypedSchema) -> int:
             raise SchemaRetrievalError(result.json())
         return result.json()["id"]
 
-    async def get_latest_schema(self, subject: str) -> (int, TypedSchema):
+    async def get_latest_schema(self, subject: str) -> Tuple[int, TypedSchema]:
         result = await self.client.get(f"subjects/{quote(subject)}/versions/latest")
         if not result.ok:
             raise SchemaRetrievalError(result.json())

requirements-dev.txt

@@ -1,13 +1,13 @@
 # linters and formatters
 isort==4.3.10
 pycodestyle==2.6.0
-pylint==2.4.4
+pylint==2.7.4
 yapf==0.30.0
 flake8==3.8.4
 mypy==0.812
 
 # testing
-pytest==6.2.2
+pytest==6.2.5
 pytest-xdist[psutil]==2.2.1
 pytest-timeout==1.4.2
 pdbpp==0.10.2

requirements.txt

@@ -1,7 +1,7 @@
 # PyPI dependencies
 accept-types==0.4.1
 aiohttp-socks==0.5.5
-aiokafka==0.6.0
+aiokafka==0.7.2
 jsonschema==3.2.0
 lz4==3.0.2
 requests==2.23.0
@@ -19,3 +19,21 @@ protobuf~=3.14.0
 #
 git+https://github.com/aiven/avro.git@513b153bac5040af6bba5847aef202adb680b67b#subdirectory=lang/py3/
 git+https://github.com/aiven/kafka-python.git@b9f2f78377d56392f61cba8856dc6c02ae841b79
+
+# Indirect dependencies
+aiohttp==3.8.1
+aiosignal==1.2.0
+async-timeout==4.0.2
+attrs==21.4.0
+certifi==2021.10.8
+chardet==3.0.4
+charset-normalizer==2.0.10
+decorator==5.1.1
+frozenlist==1.3.0
+idna==2.10
+multidict==6.0.2
+pyrsistent==0.18.1
+python-socks==2.0.3
+six==1.16.0
+urllib3==1.25.11
+yarl==1.7.2

setup.py

@@ -64,6 +64,7 @@
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Topic :: Database :: Database Engines/Servers",
         "Topic :: Software Development :: Libraries",
     ],