Drop the legacy client (#14243)

* Drop the legacy client

* Fix tests and style

---------

Co-authored-by: Andrew Zhang <[email protected]>

LICENSE-3rdparty.csv

@@ -31,14 +31,11 @@ flup-py3,Vendor,BSD-3-Clause,"Copyright (c) 2005, 2006 Allan Saddi <allan@saddi.
 foundationdb,PyPI,Apache-2.0,Copyright 2017 FoundationDB
 futures,PyPI,PSF,Copyright (c) 2015 Brian Quinlan
 gearman,PyPI,Apache-2.0,Copyright 2010 Yelp <[email protected]>
-gssapi,PyPI,ISC,"Copyright (c) 2014, The Python GSSAPI Team"
 immutables,PyPI,Apache-2.0,Copyright 2018-present Contributors to the immutables project.
 importlib-metadata,PyPI,Apache-2.0,Copyright Jason R. Coombs
 in-toto,PyPI,Apache-2.0,Copyright 2018 New York University
 ipaddress,PyPI,PSF,Copyright (c) 2013 Philipp Hagemeister
 jellyfish,PyPI,BSD-3-Clause,Copyright (c) 2015 James Turk
-kafka-python,PyPI,Apache-2.0,Copyright 2015 David Arthur
-kazoo,PyPI,Apache-2.0,Copyright 2012 Kazoo team
 kubernetes,PyPI,Apache-2.0,Copyright 2014 The Kubernetes Authors.
 ldap3,PyPI,LGPL-3.0-only,Copyright 2013 - 2020 Giovanni Cannata
 lxml,PyPI,BSD-3-Clause,Copyright (c) 2004 Infrae. All rights reserved.

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -27,16 +27,13 @@ enum34==1.1.10; python_version < '3.0'
 foundationdb==6.3.24; python_version > '3.0'
 futures==3.4.0; python_version < '3.0'
 gearman==2.0.2; sys_platform != 'win32' and python_version < '3.0'
-gssapi==1.8.2; python_version > '3.0'
 immutables==0.19; python_version > '3.0'
 importlib-metadata==2.1.3; python_version < '3.8'
 in-toto==1.0.1; python_version > '3.0'
 ipaddress==1.0.23; python_version < '3.0'
 jaydebeapi==1.2.3
 jellyfish==0.9.0; python_version > '3.0'
 jpype1==1.4.1; python_version > '3.0'
-kafka-python==2.0.2
-kazoo==2.9.0
 kubernetes==18.20.0; python_version < '3.0'
 kubernetes==26.1.0; python_version > '3.0'
 ldap3==2.9.1

kafka_consumer/assets/configuration/spec.yaml

@@ -182,14 +182,4 @@ files:
         value:
           type: string
           example: <SSL_FILE_PATH>
-      - name: broker_requests_batch_size
-        description: |
-          The OffsetRequests sent to each broker are batched by the kafka_consumer check in groups of 30 by default.
-          If the batch size is too big, you may see KafkaTimeoutError exceptions in the logs while
-          running the wakeup calls.
-          If the batch size is too small, the check will take longer to run.
-        display_default: 30
-        value:
-          type: integer
-          example: 30
       - template: instances/default

kafka_consumer/datadog_checks/kafka_consumer/client/confluent_kafka_client.py → kafka_consumer/datadog_checks/kafka_consumer/client.py

@@ -6,11 +6,16 @@
 from six import string_types
 
 from datadog_checks.base import ConfigurationError
-from datadog_checks.kafka_consumer.client.kafka_client import KafkaClient
 from datadog_checks.kafka_consumer.constants import KAFKA_INTERNAL_TOPICS
 
 
-class ConfluentKafkaClient(KafkaClient):
+class KafkaClient:
+    def __init__(self, config, tls_context, log) -> None:
+        self.config = config
+        self.log = log
+        self._kafka_client = None
+        self._tls_context = tls_context
+
     @property
     def kafka_client(self):
         if self._kafka_client is None:
@@ -71,8 +76,6 @@ def __get_authentication_config(self):
         return config
 
     def get_highwater_offsets(self, consumer_offsets):
-        # TODO: Remove broker_requests_batch_size as config after
-        # kafka-python is removed if we don't need to batch requests in Confluent
         highwater_offsets = {}
         topics_with_consumer_offset = {}
         if not self.config._monitor_all_broker_highwatermarks: