From bb62d308244226d0806dd459c7e0a98866e46d00 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Wed, 8 Mar 2023 15:31:33 +0100 Subject: [PATCH] Add a tls e2e env --- .../client/confluent_kafka_client.py | 25 ++- .../datadog_checks/kafka_consumer/config.py | 5 + kafka_consumer/hatch.toml | 8 +- kafka_consumer/tests/certificate/cert.cert | 20 -- kafka_consumer/tests/certificate/server.pem | 48 ----- kafka_consumer/tests/conftest.py | 86 ++++---- kafka_consumer/tests/docker/ssl/cert.pem | 27 +++ .../tests/docker/ssl/certificate/caroot.pem | 18 ++ .../tests/docker/ssl/certificate/cert.pem | 21 ++ .../tests/docker/ssl/certificate/key.pem | 41 ++++ kafka_consumer/tests/docker/ssl/client.config | 7 + .../tests/docker/ssl/command.properties | 7 + .../tests/docker/ssl/docker-compose.yaml | 89 +++++++++ .../tests/docker/ssl/generate-certificates.sh | 184 ++++++++++++++++++ .../ssl/keystore/client.server.keystore.jks | Bin 0 -> 5070 bytes .../ssl/keystore/kafka1.server.keystore.jks | Bin 0 -> 5070 bytes .../ssl/keystore/kafka2.server.keystore.jks | Bin 0 -> 5070 bytes .../keystore/localhost.server.keystore.jks | Bin 0 -> 5086 bytes .../tests/docker/ssl/truststore/ca-key | 28 +++ .../ssl/truststore/kafka.truststore.jks | Bin 0 -> 1126 bytes .../tests/python_client/test_unit.py | 13 -- kafka_consumer/tests/runners.py | 56 ++++-- 22 files changed, 538 insertions(+), 145 deletions(-) delete mode 100644 kafka_consumer/tests/certificate/cert.cert delete mode 100644 kafka_consumer/tests/certificate/server.pem create mode 100644 kafka_consumer/tests/docker/ssl/cert.pem create mode 100644 kafka_consumer/tests/docker/ssl/certificate/caroot.pem create mode 100644 kafka_consumer/tests/docker/ssl/certificate/cert.pem create mode 100644 kafka_consumer/tests/docker/ssl/certificate/key.pem create mode 100644 kafka_consumer/tests/docker/ssl/client.config create mode 100644 kafka_consumer/tests/docker/ssl/command.properties create mode 100644 kafka_consumer/tests/docker/ssl/docker-compose.yaml create mode 100755 kafka_consumer/tests/docker/ssl/generate-certificates.sh create mode 100644 kafka_consumer/tests/docker/ssl/keystore/client.server.keystore.jks create mode 100644 kafka_consumer/tests/docker/ssl/keystore/kafka1.server.keystore.jks create mode 100644 kafka_consumer/tests/docker/ssl/keystore/kafka2.server.keystore.jks create mode 100644 kafka_consumer/tests/docker/ssl/keystore/localhost.server.keystore.jks create mode 100644 kafka_consumer/tests/docker/ssl/truststore/ca-key create mode 100644 kafka_consumer/tests/docker/ssl/truststore/kafka.truststore.jks diff --git a/kafka_consumer/datadog_checks/kafka_consumer/client/confluent_kafka_client.py b/kafka_consumer/datadog_checks/kafka_consumer/client/confluent_kafka_client.py index fd05e472259c6..60d130ddda9ee 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/client/confluent_kafka_client.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/client/confluent_kafka_client.py @@ -14,13 +14,24 @@ class ConfluentKafkaClient(KafkaClient): @property def kafka_client(self): if self._kafka_client is None: - self._kafka_client = AdminClient( - { - "bootstrap.servers": self.config._kafka_connect_str, - "socket.timeout.ms": self.config._request_timeout_ms, - "client.id": "dd-agent", - } - ) + config = { + "bootstrap.servers": self.config._kafka_connect_str, + "socket.timeout.ms": self.config._request_timeout_ms, + } + + if self.config._use_tls: + config.update( + { + "security.protocol": "ssl", + "ssl.ca.location": self.config._tls_ca_cert, + "ssl.certificate.location": self.config._tls_cert, + "ssl.key.location": self.config._tls_private_key, + "ssl.key.password": self.config._tls_private_key_password, + } + ) + + self._kafka_client = AdminClient(config) + return self._kafka_client def create_kafka_admin_client(self): diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index 54a6d11dcdafb..99d8524a91dea 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -38,6 +38,11 @@ def __init__(self, init_config, instance) -> None: self._sasl_kerberos_service_name = instance.get('sasl_kerberos_service_name', 'kafka') self._sasl_kerberos_domain_name = instance.get('sasl_kerberos_domain_name') self._sasl_oauth_token_provider = instance.get('sasl_oauth_token_provider') + self._use_tls = instance.get('use_tls', False) + self._tls_ca_cert = instance.get("tls_ca_cert") + self._tls_cert = instance.get("tls_cert") + self._tls_private_key = instance.get("tls_private_key") + self._tls_private_key_password = instance.get("tls_private_key_password") self.use_legacy_client = is_affirmative(instance.get('use_legacy_client', False)) def validate_config(self): diff --git a/kafka_consumer/hatch.toml b/kafka_consumer/hatch.toml index 5e63ef4b5ed7b..7a15b40bdec59 100644 --- a/kafka_consumer/hatch.toml +++ b/kafka_consumer/hatch.toml @@ -20,10 +20,10 @@ impl = ["legacy"] python = ["3.8"] version = ["1.1", "2.3", "3.3"] -#[[envs.default.matrix]] -#python = ["3.8"] -#version = ["3.3"] -#auth = ["ssl"] +[[envs.default.matrix]] +python = ["3.8"] +version = ["3.3"] +auth = ["ssl"] [envs.default.overrides] matrix.version.e2e-env = { value = true, if = ["3.3"] } diff --git a/kafka_consumer/tests/certificate/cert.cert b/kafka_consumer/tests/certificate/cert.cert deleted file mode 100644 index 2b7e72ccc5322..0000000000000 --- a/kafka_consumer/tests/certificate/cert.cert +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDQjCCAioCCQC6nf2EOioOfjANBgkqhkiG9w0BAQsFADBjMQswCQYDVQQGEwJG -UjEKMAgGA1UECAwBQTEKMAgGA1UEBwwBQTEKMAgGA1UECgwBQTEKMAgGA1UECwwB -QTESMBAGA1UEAwwJbG9jYWxob3N0MRAwDgYJKoZIhvcNAQkBFgFBMB4XDTE4MDEy -OTEwNDMwMFoXDTI4MDEyNzEwNDMwMFowYzELMAkGA1UEBhMCRlIxCjAIBgNVBAgM -AUExCjAIBgNVBAcMAUExCjAIBgNVBAoMAUExCjAIBgNVBAsMAUExEjAQBgNVBAMM -CWxvY2FsaG9zdDEQMA4GCSqGSIb3DQEJARYBQTCCASIwDQYJKoZIhvcNAQEBBQAD -ggEPADCCAQoCggEBAMnJLZ9gYuHDfWH4oLace6k7CkUSKy2sUFkSmQpTo6BX3yGm -heDpyhaqCbr8tCs8n+GkJlL6jVr7CpmE4anDl6VOqzFWJra12isU6aYdN1rm59YS -kKvhw74jW68Hy3RhLeJVBZHA20OSYKRwHPVIiTnGWF06hi99O2KHhAK2gUFLul/u -OJSt2vbwkqTRQ8vL4wqikZfkIfoyUkAoOkUwKh25yvutQqg9r31QUO2uKunFJQox -9j4lHbbR31KwnvBx1/MNi4gZjB6P/rHNxT6oxRTdX4b47Ozs6FGDuDCZzk84bA59 -IsGaAzFTGnaN6LxwHbdPDm8/WXrAOJBBXmRcaKcCAwEAATANBgkqhkiG9w0BAQsF -AAOCAQEAWmo97vgirkk7rtjflx59YTPjvhZnaz9d+jt+3U0DL754kYJjUV5w/LO4 -qhhJejqYF1oQWnFVRRRlaPpmT33i6P3HBGLajKJDYYO1B05FTJ+BUYkLa9uhmJEu -0mhyxhVtvGjuVkEdA19qrqNgY9YZiLgtlqT324UVWlZRglc5+BIN3foh2vQNltdr -TJBEJc7UCGyM2zt7LJQrmXt7ukX31b3NNhRLG+HdfY5u4iHcbIMIeOyT4DgwrHqb -EJuAkrsRk07ZEXGHjxf604AXHgtPraRNLljzllDTvoDYsYYh0MQ8rC4nOxyY+HqF -f1b7t7dbaca+7brsbAQxMGHIyonS6g== ------END CERTIFICATE----- diff --git a/kafka_consumer/tests/certificate/server.pem b/kafka_consumer/tests/certificate/server.pem deleted file mode 100644 index 55643a514e633..0000000000000 --- a/kafka_consumer/tests/certificate/server.pem +++ /dev/null @@ -1,48 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDJyS2fYGLhw31h -+KC2nHupOwpFEistrFBZEpkKU6OgV98hpoXg6coWqgm6/LQrPJ/hpCZS+o1a+wqZ -hOGpw5elTqsxVia2tdorFOmmHTda5ufWEpCr4cO+I1uvB8t0YS3iVQWRwNtDkmCk -cBz1SIk5xlhdOoYvfTtih4QCtoFBS7pf7jiUrdr28JKk0UPLy+MKopGX5CH6MlJA -KDpFMCoducr7rUKoPa99UFDtrirpxSUKMfY+JR220d9SsJ7wcdfzDYuIGYwej/6x -zcU+qMUU3V+G+Ozs7OhRg7gwmc5POGwOfSLBmgMxUxp2jei8cB23Tw5vP1l6wDiQ -QV5kXGinAgMBAAECggEATM7JEOtteCYjadFry1bRM1+HzzbVPLNkeiM8IdQoanZI -NtTeDPiwtePPhF+AbRzMduU2uUsp10HLjRgplacf6AqD22nBnral7tQtA9AHLBDM -mo3Ch8qN6agn0/4RkaEUCBz427wfOh9TrgQG4c7UfStko0gNn7H5ks2mkCnMFYAC -zV3aIVm78O/uzlLnQh+t1qgVHGWi8flGNosjTlaW3R0tTvXjA9bWgBvZPunkUOzL -BU3uS3SyBdrZobPIX/MAfuUHaDUkoCSgE5rFXKDYamHMST1f+NNb8UxMtCsnhMVX -9pXyGeIeJsy1qTjDbxfPoPxihjEYlnKXPAOOwPbbQQKBgQDl2rOThg7zHSdcKNRn -sGqHNF1D+1BKwxfXuVmjiIoVkg8iaaVRIIeHQn1K3Ma5M3IUsvcf+RYOXjD/b9P2 -8PPbHhUrq/5Vi4Okdh4PJw3+d67vqn3IGw8oa6PkhNU66YHuqILE3lHmZhq2ot0h -fPynCl5CkFSZJ3+UCDEwQ2LMFwKBgQDgvSFyL3tIXqlsW6o2bItfV9y2q7d/ig9/ -BfY+TY02hDIDGhMWpBPjZugw3zk4lair34WsPgJCE6Md8jMam8/Vh2SjIDiN7uxA -Cq+jymkTYgcwBRonZ19P1A6FCZYHoJvcvA7XzdofSdN3knB4BoD2D9jj+x8PTrmd -yxZpZdFR8QKBgB0cMZoBZGtNyepWEXCAWz18WEN/1I9jXeWoR736UOKieOA22QUJ -RaDPvuZ4R4K1DUGp2Lpn5W80ZqgzFXk00u1xReUeBzv2S1nMy8nHfHI53iHkScHK -C4Vm/H64g/jOEfKTzNv5/A1d6QwwXCkdlzHgkkeD1/L9mZ652uDt59lJAoGAKiz0 -OOSWQ0eYjgBJ2jrWkQcBIbrqDeJP9CufQaP2K1P1usVK7cXPAa8PRhSuPWNAqzig -cmmUrOfYhcctNgIlp9Hn3gEu+9vptD8NJTTYMU4FQUrD1L8+kNGgA/F/mWDXvz3/ -nxo/o/2L1PXDa5L04riooGhBNPM2PFmflBoKhVECgYEApksK/s6hc/Jtse0OPN+s -wx6JHWUUpzmIEqeVxQVIsEHp6TozTBD1R0fGFKekgFORXMRkRdlcXFMu8zdOZZDD -x7gLBwHSgNA/OcQ+beCPWzalUGfVEv15fgdIbJFOo96+AnTv7scSpINbt7dfvuCU -+efuvbI8ha7biubPtngr97o= ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDQjCCAioCCQC6nf2EOioOfjANBgkqhkiG9w0BAQsFADBjMQswCQYDVQQGEwJG -UjEKMAgGA1UECAwBQTEKMAgGA1UEBwwBQTEKMAgGA1UECgwBQTEKMAgGA1UECwwB -QTESMBAGA1UEAwwJbG9jYWxob3N0MRAwDgYJKoZIhvcNAQkBFgFBMB4XDTE4MDEy -OTEwNDMwMFoXDTI4MDEyNzEwNDMwMFowYzELMAkGA1UEBhMCRlIxCjAIBgNVBAgM -AUExCjAIBgNVBAcMAUExCjAIBgNVBAoMAUExCjAIBgNVBAsMAUExEjAQBgNVBAMM -CWxvY2FsaG9zdDEQMA4GCSqGSIb3DQEJARYBQTCCASIwDQYJKoZIhvcNAQEBBQAD -ggEPADCCAQoCggEBAMnJLZ9gYuHDfWH4oLace6k7CkUSKy2sUFkSmQpTo6BX3yGm -heDpyhaqCbr8tCs8n+GkJlL6jVr7CpmE4anDl6VOqzFWJra12isU6aYdN1rm59YS -kKvhw74jW68Hy3RhLeJVBZHA20OSYKRwHPVIiTnGWF06hi99O2KHhAK2gUFLul/u -OJSt2vbwkqTRQ8vL4wqikZfkIfoyUkAoOkUwKh25yvutQqg9r31QUO2uKunFJQox -9j4lHbbR31KwnvBx1/MNi4gZjB6P/rHNxT6oxRTdX4b47Ozs6FGDuDCZzk84bA59 -IsGaAzFTGnaN6LxwHbdPDm8/WXrAOJBBXmRcaKcCAwEAATANBgkqhkiG9w0BAQsF -AAOCAQEAWmo97vgirkk7rtjflx59YTPjvhZnaz9d+jt+3U0DL754kYJjUV5w/LO4 -qhhJejqYF1oQWnFVRRRlaPpmT33i6P3HBGLajKJDYYO1B05FTJ+BUYkLa9uhmJEu -0mhyxhVtvGjuVkEdA19qrqNgY9YZiLgtlqT324UVWlZRglc5+BIN3foh2vQNltdr -TJBEJc7UCGyM2zt7LJQrmXt7ukX31b3NNhRLG+HdfY5u4iHcbIMIeOyT4DgwrHqb -EJuAkrsRk07ZEXGHjxf604AXHgtPraRNLljzllDTvoDYsYYh0MQ8rC4nOxyY+HqF -f1b7t7dbaca+7brsbAQxMGHIyonS6g== ------END CERTIFICATE----- diff --git a/kafka_consumer/tests/conftest.py b/kafka_consumer/tests/conftest.py index a2d410cac4f61..8a4164876931e 100644 --- a/kafka_consumer/tests/conftest.py +++ b/kafka_consumer/tests/conftest.py @@ -13,14 +13,14 @@ from datadog_checks.dev import WaitFor, docker_run from datadog_checks.kafka_consumer import KafkaCheck -from .common import DOCKER_IMAGE_PATH, HERE, HOST_IP, KAFKA_CONNECT_STR, LEGACY_CLIENT, TOPICS +from .common import AUTHENTICATION, DOCKER_IMAGE_PATH, HERE, HOST_IP, KAFKA_CONNECT_STR, LEGACY_CLIENT, TOPICS from .runners import KConsumer, Producer -# Dummy TLS certs -CERTIFICATE_DIR = os.path.join(os.path.dirname(__file__), 'certificate') -cert = os.path.join(CERTIFICATE_DIR, 'cert.cert') -private_key = os.path.join(CERTIFICATE_DIR, 'server.pem') - +CERTIFICATE_DIR = os.path.join(os.path.dirname(__file__), 'docker', 'ssl', 'certificate') +ROOT_CERTIFICATE = os.path.join(CERTIFICATE_DIR, 'caroot.pem') +CERTIFICATE = os.path.join(CERTIFICATE_DIR, 'cert.pem') +PRIVATE_KEY = os.path.join(CERTIFICATE_DIR, 'key.pem') +PRIVATE_KEY_PASSWORD = 'secret' if LEGACY_CLIENT: E2E_METADATA = { @@ -37,13 +37,28 @@ 'start_commands': ['bash /tmp/start_commands.sh'], } -INSTANCE = { - 'kafka_connect_str': KAFKA_CONNECT_STR, - 'tags': ['optional:tag1'], - 'consumer_groups': {'my_consumer': {'marvel': [0]}}, - 'broker_requests_batch_size': 1, - 'use_legacy_client': LEGACY_CLIENT, -} +if AUTHENTICATION == "ssl": + INSTANCE = { + 'kafka_connect_str': "kafka1:9092", + 'tags': ['optional:tag1'], + 'consumer_groups': {'my_consumer': {'marvel': [0]}}, + 'broker_requests_batch_size': 1, + 'use_tls': True, + 'tls_validate_hostname': True, + 'tls_cert': CERTIFICATE, + 'tls_private_key': PRIVATE_KEY, + 'tls_private_key_password': PRIVATE_KEY_PASSWORD, + 'tls_ca_cert': ROOT_CERTIFICATE, + 'use_legacy_client': LEGACY_CLIENT, + } +else: + INSTANCE = { + 'kafka_connect_str': KAFKA_CONNECT_STR, + 'tags': ['optional:tag1'], + 'consumer_groups': {'my_consumer': {'marvel': [0]}}, + 'broker_requests_batch_size': 1, + 'use_legacy_client': LEGACY_CLIENT, + } @pytest.fixture(scope='session') @@ -80,22 +95,6 @@ def kafka_instance(): return copy.deepcopy(INSTANCE) -@pytest.fixture -def kafka_instance_tls(): - return { - 'kafka_connect_str': KAFKA_CONNECT_STR, - 'tags': ['optional:tag1'], - 'consumer_groups': {'my_consumer': {'marvel': [0]}}, - 'broker_requests_batch_size': 1, - 'use_tls': True, - 'tls_validate_hostname': True, - 'tls_cert': cert, - 'tls_private_key': private_key, - 'tls_ca_cert': CERTIFICATE_DIR, - 'use_legacy_client': LEGACY_CLIENT, - } - - def create_topics(): client = _create_admin_client() @@ -106,10 +105,8 @@ def create_topics(): def initialize_topics(): - consumer = KConsumer(TOPICS) - - with Producer(): - with consumer: + with Producer(INSTANCE): + with KConsumer(INSTANCE, TOPICS): time.sleep(5) @@ -121,9 +118,20 @@ def mock_local_kafka_hosts_dns(): def _create_admin_client(): - return AdminClient( - { - "bootstrap.servers": KAFKA_CONNECT_STR, - "socket.timeout.ms": 1000, - } - ) + config = { + "bootstrap.servers": INSTANCE['kafka_connect_str'], + "socket.timeout.ms": 1000, + } + + if INSTANCE.get('use_tls', False): + config.update( + { + "security.protocol": "ssl", + "ssl.ca.location": INSTANCE.get("tls_ca_cert"), + "ssl.certificate.location": INSTANCE.get("tls_cert"), + "ssl.key.location": INSTANCE.get("tls_private_key"), + "ssl.key.password": INSTANCE.get("tls_private_key_password"), + } + ) + + return AdminClient(config) diff --git a/kafka_consumer/tests/docker/ssl/cert.pem b/kafka_consumer/tests/docker/ssl/cert.pem new file mode 100644 index 0000000000000..3f5878f800537 --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEnjCCAoYCCQCg3s7NDJiDPDANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZj +bGllbnQwHhcNMjMwMzA4MTQyODIxWhcNMzMwMzA1MTQyODIxWjARMQ8wDQYDVQQD +DAZjbGllbnQwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCvTeOWMVla +oePAw4FywBSe22dGXhvHxCzYQ8cE2rWb4NkLfr+nq428hPd2wtI+lhnXy+XpSoRk +r+D02oNtkk/U/tBQxutnMvo4jy6g+N5HltfTwLrNgPROVbDZRRonv3CJp96fMdFc +2fpdD5KKRLgqGogL1aZIib7nxdadRC5oHRGJLszxGrTygYsep9QrJWY7S4c9LxTS +DbS7Dtg9VaCuu97U71Dg8fQo/1hGWcqQRn8OvBz649TZ3cRyPph8iLlj/ZAwcMcE +rTGK0Bd3wR7nvVpqCbpx1Sn4z8dYaB+0/7Oiq30EzW9XHTKyuI2L3aqpK8j4Jt4H +gM+L4nobM6ja68yu5b/5Ddms5SK81oVlXp7SnLA3E6A+HIAiAN7LCGBsbucPE2BI +oAuYxJuha23I+gBa8ZOF4p/j7EQK4au3Hx3HLpk11tE+qMikYQzYnGblzb6pWbzv +KS37zkkNhlwUn7Dr4YxMwB/usuaJEck/CI7QaJ7Jy7lpZq/6qSUfN5iHDcZqDpTF +16Rvf+2BsZ3D7/xlmcBGc5BE3aR/XkJehK9vT42/2IUesxEk6ZNrjqQVy5fXtVuu +uhlB+q3A1Wwq2fy/j4pU4nuYG6a9+XB2845vGoxk/KbTZ2Fg1h5OGXynlfgd6nSC +hJrxya12ykeIkPoL2Q4Jnai+M347U8fdQQIDAQABMA0GCSqGSIb3DQEBCwUAA4IC +AQAFSt19Pp1Jo3fd/GYPBdNtXd1rLIVoDlDyLFnouylExMu0UxNwOxVu5qRsWoNr +FSmecY2YuCjgyy0QmuR51zcfi2pV7HiUh00SJgPwa71yBRQD5nlI2aXdnWj1k+mW +ygJWfGw6wIQ1WiE4qDgHpcdlbzLry03kts8cN/Kryos+F5TZ1QEBGQREPLY06aTZ +fZwYPOofUs5Ckq4JKQkcAvOoZjjy1IuPlvKjQFVfFfoHCDmiAzshrrxxdJr1Vz7u +P9pA6do+MdIsLHa6E00hANTO4AaoXJx32IGCFwQb5vNw8Ak2JexIby4Xtjvigz+x +HFojc0wjGkKLA4WlqvLhd2USDBgUnsFum9ifjSJ0L8aqTo2LDcXFlp/uH3whIzOi +gY0kptE/ucdGLErMff2qdticPSiPxtHVstiZjsDS0HdpHJgsI4RDnU9Cxx7sObUV +ox0finvBCbYdSxo8vaS21LoXg2tC7cU+jlSBOD82Cz2dH3Q/2AiugSda5Zwpakpy +IrOj1xbxJZum/koRp+L5dAIFcAU3U50Lv1Ykqa3cu25J+dnqSBEKbmwMlcNs2c5C +kTfVmprALKinZgZEmNcL9lWQcB7ekGgpfe65cDQSQYYNPzeS/BSOliBEXyR2pQmE +3QfQzsDbbe9AIS0Q+4bRC3KhH0dB1IX2EZHtwf210nuYnQ== +-----END CERTIFICATE----- diff --git a/kafka_consumer/tests/docker/ssl/certificate/caroot.pem b/kafka_consumer/tests/docker/ssl/certificate/caroot.pem new file mode 100644 index 0000000000000..b1cba0ca2d1f9 --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/certificate/caroot.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC8DCCAdgCCQD1NF3JgzmSbDANBgkqhkiG9w0BAQsFADA6MQswCQYDVQQGEwJV +UzEVMBMGA1UECgwMU2VydmljZVVzZXJzMRQwEgYDVQQDDAtrYWZrYS1hZG1pbjAe +Fw0yMzAzMDkxMDQ0NTRaFw0zMzAzMDYxMDQ0NTRaMDoxCzAJBgNVBAYTAlVTMRUw +EwYDVQQKDAxTZXJ2aWNlVXNlcnMxFDASBgNVBAMMC2thZmthLWFkbWluMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3j6xm1bkA8rt8V+ZhXosc0zljA3w +HilR6ApToaGjdM9BMM2qgQwmZlIr/fjanL4KYNKNGazn+3hw8mm+JSmBPwczjgjy +EfXxJvI8XA+lDwoM7LVv9vIR4X0/QvAHvFLQuYC9kNHs2kN0xTqw+ZdztKkWFZoJ +OLygp/qglbN+0eVaS6w91PlX6URaCpjrB1QPWzUcAfvYU4D4lPeSlB5HYxSNr9Cx +RY6S66xdCk0LvkTJUEPb9/whckuJGjGOJNK8QMASPYDv/bF7U8bq5eIal6hAqVXP +91BSYCWVHs+Bb6Xt9g812vMUjLPtejlu2c4GCZ6xA/yMJ7s1Uxth9ThAVwIDAQAB +MA0GCSqGSIb3DQEBCwUAA4IBAQC9zDMd0KxuP7puG5HTWH0Uv8P/8g5b7l8xP2sW +GgEtVlMFQUS4nm6WxZrSkYbEsuEvG8URjbqFRli+2SSV8UVB3HxlkA1wYX90ForJ +/yyINj1ohqzKdsMMco6jhK9UP0opEXp4sMKBl359mjnPbu9RqjwangkhPrLZNZNF +pbo6HIY0vFlUqc0yWYAFmY8ixi3zAUKAr94eFWJwpFH7rncFYx+3Q1y5JgHvlQKT +wjjTCr7sKfAsTqUC8OE7HguS6SVKx7RRHV9C0OTL5ekdEHURC4uifgGV3WtkVQGH +5wdC0nNY/Wjuc66TKkqzLoyydfRSXpWuEpOah0FcQJX9DfLy +-----END CERTIFICATE----- diff --git a/kafka_consumer/tests/docker/ssl/certificate/cert.pem b/kafka_consumer/tests/docker/ssl/certificate/cert.pem new file mode 100644 index 0000000000000..f098611e7a023 --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/certificate/cert.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhDCCAmwCCQDY21uQbeDVTTANBgkqhkiG9w0BAQsFADA6MQswCQYDVQQGEwJV +UzEVMBMGA1UECgwMU2VydmljZVVzZXJzMRQwEgYDVQQDDAtrYWZrYS1hZG1pbjAe +Fw0yMzAzMDkxMDQ1MDFaFw0zMzAzMDYxMDQ1MDFaME4xEjAQBgNVBAMTCWxvY2Fs +aG9zdDEVMBMGA1UEChMMU2VydmljZVVzZXJzMQkwBwYDVQQHEwAxCTAHBgNVBAgT +ADELMAkGA1UEBhMCVVMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCz +KM9rFto8r4XD8iEasbSx0/hpWxNYc9I3vtasLPsedt6MY0DbXigEXHX92nx1e+Cl +qVW9EGkIzD0Jgz55pqwi0wctgTJNTmUjSkvefMxggWIUN4QTeCV9jtqZw5jN1tgS +eIT0uCkYtUvGPajEAOjWWLjmChFJkNDnI2tIQMXU/W2Ob1YnZeGfxG8qlK53bpIJ +XQY1l65LSQtUI6aDsrysjTDmYnVqHF4/FZzym2IhESIWrZQGE1/A0DJaZEO9hyn2 +//GWtnGzkt4s2+7bbuiSNaA6af9YbeDroKt+ByHXB9YAyxind1tVhNAIv4MQ6GE7 +yco01VV+P+vj9LSSCjU3b8pUrZDJEKYUth/5XSEfAxLaej74TVk6bPrZR6s03C9n +V41DKTKqPFb9Ql+s5XewRnRLp7slX2TVzbHUuLTA/ID9U68lDxOQc9mRQWt10X4o +yxqpUmsXf7u2PedO7Wl5fTRZpQsidtukYBLezYEZPpPiSP8G2FMP1f2VgvG9YyUC +AwEAATANBgkqhkiG9w0BAQsFAAOCAQEAmohuyZjCxPw2mdZrvxDMf6Rcli6DyyKL +IrPq/EH3kNylxhYix86QUNggnJZv+HSOEYVf++cPf9VamGVQZapYLMCzJVrOdNgT +hDdy9+a8Q2heLHPfJ7u11cjO6V5q4WOLEmCx4pI2cJ+Y+e3wxSKjas9A6VQaJuFm +2uHd3T6ny0m7HLkUAoM1/LqnMvGmG7wXBrsK/AwaMjeyW04skkhGks1GbmJsx/G1 +QvhnBcSJvXVjBa6PQ52e3BDpoqLAs4BuOeWuZ03D55/Cp5bYkGbo3QpWIArLuikN +yktIeUgnBvSKRTgI9GDP+uKesG/Ny8Mle0t5BbU1fXgvw/LfJNslCQ== +-----END CERTIFICATE----- diff --git a/kafka_consumer/tests/docker/ssl/certificate/key.pem b/kafka_consumer/tests/docker/ssl/certificate/key.pem new file mode 100644 index 0000000000000..edf724442c8dc --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/certificate/key.pem @@ -0,0 +1,41 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHKjAcBgoqhkiG9w0BDAEEMA4ECGwj+87fxFROAgIDJASCBwgRUvWC2+3lrnJ7 +LyZmkR1sc9Dt+EVZC3H46nIvU65PCbXotOenQ2l0RcXsWSmAXr0IQkrKU2e/TRlG +VqfTNYPmysxNv39dGjOL7aaa/Fe2i5//40Z2DaZ+OhXhg3eVYthbfsokngiyWg5O +CMez0JjHD3ZgvajUs/rK4NDDpSyRG7iGnROGTaoyOhNb3RFXqvGCf8AlR1uEGB5V +fh4CIuvgOx26YtnNtmRHTwctFbOSRmZd1eMOYWrRYAhAS+NAP1f58tezcjkYaDTf +texJurkY8hjNdFCpmhWqxVKaorcQk3nqehc8ij9ayskBqQmf5cWwiW+MVOXBkArx +mmN4LS63VKyPeTdV312a5kuEYZurdMETN9O41h3PDaLMfayqAinQVdUv1GVdPYs0 +BIuHpa+FRaqVM5RZugjrj3AzmVtwfKsAdM/r9OnauFpdiWrWvHIhfKCc1ZibwmSY +X4DagVMEs9t/jPSxBgEfb0RKc9UbHatIrm1hG6UDj5fbhPvMOz7xlaDxiOgXwUB0 +6HyDc1xQqDLLgN2mFwFJ0okZUaW4gfpF7EjCzGFfMImTkkive8GbH0rbaUFIaRqK +SzcoP4hDUyxC1Iiz1k4OeiOG/fKai9RNeU4v3muyRoXE8FW5JtgT8yRKTgilrC/C +yn6cpI6wawgn62FGn4r9slXlbbLd0Q0MI1LEbPSvYo14EDBokfmZrosAHHzoC7xT +dy1tDQEK66UGbZ/CDI55dHXasGPAajkTa0tPjPFcoJjqkZs/9uBpgGT2b3Y9FVyH +nI5cO907ST8rPJNmHKYulf0nV4VtQiMvewSFbDQiz8/Qyb3KxkwY93WC4ia/wAZH ++r1eUyZM6cRR3dug+vHWqmX1Amw4ogLJax+rJbdCVXHA3SQlsPe2YcbX8axdKrTq +EWPzYqx8ZEgtdi/aVcyRh/sfoMUv6bUmJg6eVH+3ph1KKlBT2AZU9zPuJu2tkJUW +Ols+gscp1TU6BVON+vRRLHyhu7yz2jx0WbyNLtaCDaEHTEaXk8McCWp70kimqz/T +jLHahVZ8YJprtksSb9mXSqgiJzTOFMYv03cf3F1PNB928KZWGhU/trDis2WJB5FC +9E7WCTFVKr21+onoQFcZ86mBHzlATUu8BXd0LTWvjSqlHhfynyNyk0P0QjMwkin4 +9V9SWHSGimD6YENM9o+zyp9DgGvEuRMUztvrxZ2lVVsBZMCCQhja/wakeIOVwQDx +S8VRjKm8/DgAYZHUCJTvOIBxLnodImO7VdnpGpaknT9v5E2aWyWFs8RjpeP7X7Pa +95r62IKMstvRam4QvXX7ascO7uI4vNpl/RkGSw9gKjj+rqH7IZ37JuDskR7HZTsp +vFYkUtcdoBkZUeQoaabzj8zCXTvhUvbdWHep+IOGnEJ54cYDzxmsDYsdveTHncT2 +GkPZGWFQfbh2wX1V7AHxdKsbwY8kWWaGj1HdNzKtoCqFQmBj9TGqHuX6FX1SzWzk +Fw1eHLJ53I9OHQOyRtpmKknQbNB5AASVSoMYQx7Qj4/BKGNsUYt6ymyarNsW4e0g +k/e14lH4uKhRvUuUaJUe7nNVjm4Q888lOWQTMmawYaobtOv+wAh9FD1E5ThvQ4d2 +dtWaNLvGost69TWm+BqUai4JJ/Ws/t8uZzws2sbbwFO7FdMEOSniYT460HlxHWfo +Zl3BFvxbSmsFwKCCAhtuGEHi56WC27wD+yVEEBDdOpqkoltBQw8hRjhNMozDb1we +EzGyaCHfXtOlKMomJzODW0Erb8PbR+m50CHYLhu5E49CoJ6uhWyFJzKVppYxhQC6 +kbv2b25TONLtwW0/EuIPtIpcI3k4MewJbiEFtOxzhpU8p0fosPJvJLmbbiM5oKsv +Z7J09LMxQfn+rILlHeywhjaYtCodqnP/1lfvX6Cwzc1QSI6j5A6pe4IFMPfarlY6 +XX4+0eqpsFQSrJGvXh6+lfwZ1SKvjQi2+LOYz2+Ia0wsqlr+RxhjyrrHLN3hheCN +GFC7zXldMaa62Gk9BGnEF626yRnDU3MSe3WyOPl1KLCZLjFhGGo1h56kf6bQGpKj +d/Ai7YMdHJo3d0eOo+w/3cgIdHbdILIXpBRXwxk8lSMfkkkQXinvhUF2XMsR2kNl +djSByS1Dh7oBibM+llN3Ole36GVU2QAxXZImnKZctJMi+7dy1cXtVfX5Tdie3To3 +w2vcwVCa81iUAAv+WjwlB2Oit5OIVpvRoCm6JX07u5UiL7iWmnhyW/Zuh6Rojbl3 +FgRezdkSCwWxMCPXF+MU3O5EUHSbqrIFmIcMUEydeMJakzKFmugjwkdSObllnbUu +WiGAjmrcYu7x7Fk59kjk20BIsrwBCH29+HiSj/eDmEu+zeuOV/ntJoZ8hf/td+al +g4B0DD3hTUdDJo+Cd70= +-----END ENCRYPTED PRIVATE KEY----- diff --git a/kafka_consumer/tests/docker/ssl/client.config b/kafka_consumer/tests/docker/ssl/client.config new file mode 100644 index 0000000000000..735e45bb26c77 --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/client.config @@ -0,0 +1,7 @@ +bootstrap.servers=kafka1:9092 +security.protocol=SSL +ssl.truststore.location=/bitnami/kafka/config/certs/kafka.truststore.jks +ssl.truststore.password=secret +ssl.keystore.location=/bitnami/kafka/config/certs/kafka.keystore.jks +ssl.keystore.password=secret +ssl.key.password=secret \ No newline at end of file diff --git a/kafka_consumer/tests/docker/ssl/command.properties b/kafka_consumer/tests/docker/ssl/command.properties new file mode 100644 index 0000000000000..92fa0264ae31a --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/command.properties @@ -0,0 +1,7 @@ +bootstrap.servers=kafka1:19092 +security.protocol=SSL +ssl.truststore.location=/bitnami/kafka/config/certs/kafka.truststore.jks +ssl.truststore.password=secret +ssl.keystore.location=/bitnami/kafka/config/certs/kafka.keystore.jks +ssl.keystore.password=secret +ssl.key.password=secret diff --git a/kafka_consumer/tests/docker/ssl/docker-compose.yaml b/kafka_consumer/tests/docker/ssl/docker-compose.yaml new file mode 100644 index 0000000000000..c95bea4f80b04 --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/docker-compose.yaml @@ -0,0 +1,89 @@ +version: '3.9' + +services: + zookeeper: + image: docker.io/bitnami/zookeeper:${ZK_VERSION} + ports: + - 2181:2181 + container_name: zookeeper + hostname: zookeeper + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + healthcheck: + test: nc -vz localhost 2181 || exit -1 + start_period: 5s + interval: 5s + timeout: 5s + retries: 10 + + kafka1: + image: docker.io/bitnami/kafka:${KAFKA_VERSION} + container_name: kafka1 + hostname: kafka1 + ports: + - "9092:9092" + environment: + KAFKA_CFG_BROKER_ID: 1 + KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_CFG_LISTENERS: INTERNAL://:19092,EXTERNAL://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://:19092,EXTERNAL://:9092 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:SSL,EXTERNAL:SSL + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_CFG_SSL_TRUSTSTORE_TYPE: jks + KAFKA_CFG_SSL_TRUSTSTORE_PASSWORD: secret + KAFKA_CFG_SSL_KEYSTORE_PASSWORD: secret + KAFKA_CFG_SSL_KEY_PASSWORD: secret + KAFKA_CFG_SSL_CLIENT_AUTH: required + KAFKA_CFG_SSL_KEYSTORE_TYPE: jks + KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: HTTPS + KAFKA_CFG_SUPER_USERS: User:kafka + KAFKA_CFG_MIN_INSYNC_REPLICAS: 2 + KAFKA_CFG_MESSAGE_MAX_BYTES: 1000000 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: false + KAFKA_CFG_SSL_PRINCIPAL_MAPPING_RULES: 'RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/,RULE:^CN=(.*?),OU=(.*?),O=(.*?),L=(.*?),ST=(.*?),C=(.*?)$/$1@$2/L,RULE:^.*[Cc][Nn]=([a-zA-Z0-9.]*).*$/$1/L,DEFAULT' + ALLOW_PLAINTEXT_LISTENER: yes + volumes: + - ./truststore/kafka.truststore.jks:/bitnami/kafka/config/certs/kafka.truststore.jks + - ./keystore/kafka1.server.keystore.jks:/bitnami/kafka/config/certs/kafka.keystore.jks + depends_on: + zookeeper: + condition: service_healthy + + kafka2: + image: docker.io/bitnami/kafka:${KAFKA_VERSION} + container_name: kafka2 + hostname: kafka2 + ports: + - "9093:9093" + environment: + KAFKA_CFG_BROKER_ID: 2 + KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_CFG_LISTENERS: INTERNAL://:19093,EXTERNAL://:9093 + KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://:19093,EXTERNAL://:9093 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:SSL,EXTERNAL:SSL + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_CFG_SSL_TRUSTSTORE_TYPE: jks + KAFKA_CFG_SSL_TRUSTSTORE_PASSWORD: secret + KAFKA_CFG_SSL_KEYSTORE_PASSWORD: secret + KAFKA_CFG_SSL_KEY_PASSWORD: secret + KAFKA_CFG_SSL_CLIENT_AUTH: required + KAFKA_CFG_SSL_KEYSTORE_TYPE: jks + KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: HTTPS + KAFKA_CFG_SUPER_USERS: User:kafka + KAFKA_CFG_MIN_INSYNC_REPLICAS: 2 + KAFKA_CFG_MESSAGE_MAX_BYTES: 1000000 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: false + KAFKA_CFG_SSL_PRINCIPAL_MAPPING_RULES: 'RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/,RULE:^CN=(.*?),OU=(.*?),O=(.*?),L=(.*?),ST=(.*?),C=(.*?)$/$1@$2/L,RULE:^.*[Cc][Nn]=([a-zA-Z0-9.]*).*$/$1/L,DEFAULT' + ALLOW_PLAINTEXT_LISTENER: yes + volumes: + - ./truststore/kafka.truststore.jks:/bitnami/kafka/config/certs/kafka.truststore.jks + - ./keystore/kafka2.server.keystore.jks:/bitnami/kafka/config/certs/kafka.keystore.jks + depends_on: + zookeeper: + condition: service_healthy diff --git a/kafka_consumer/tests/docker/ssl/generate-certificates.sh b/kafka_consumer/tests/docker/ssl/generate-certificates.sh new file mode 100755 index 0000000000000..6c93a369f97d3 --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/generate-certificates.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash + +set -eu + +export COUNTRY=US +export STATE= +export ORGANIZATION_UNIT=ServiceUsers +export CITY= +export PASSWORD=secret +export CN=kafka-admin + +validity=3650 +defaultTruststoreName="kafka.truststore.jks" +truststoreWorkDir="truststore" +keystoreWorkDir="keystore" +caKey="ca-cert" +keystoreSignInRequest="cert-file" +keystoreSignRequestSrl="ca-cert.srl" +keystoreSignedCert="cert-signed" + +country=$COUNTRY +state=$STATE +OU=$ORGANIZATION_UNIT +CN=$CN +location=$CITY +password=$PASSWORD + +function file_exists_and_exit() { +echo "'$1' cannot exist. Move or delete it before" +echo "re-running this script." +exit 1 +} + +if [ -e "$keystoreWorkDir" ]; then + file_exists_and_exit $keystoreWorkDir +fi + +if [ -e "$caKey" ]; then + file_exists_and_exit $caKey +fi + +if [ -e "$keystoreSignInRequest" ]; then + file_exists_and_exit $keystoreSignInRequest +fi + +if [ -e "$keystoreSignRequestSrl" ]; then + file_exists_and_exit $keystoreSignRequestSrl +fi + +if [ -e "$keystoreSignedCert" ]; then + file_exists_and_exit $keystoreSignedCert +fi + +echo "Welcome to the Kafka SSL keystore and trust store generator script." + +truststoreFile="" +truststorePrivateKey="" + +if [ ! -e "$truststoreWorkDir" ]; then + mkdir $truststoreWorkDir + echo + echo "OK, we'll generate a trust store and associated private key." + echo + echo "First, the private key." + echo + + openssl req -new -x509 -keyout $truststoreWorkDir/ca-key \ + -out $truststoreWorkDir/ca-cert -days $validity -nodes \ + -subj "/C=$country/ST=$state/L=$location/O=$OU/CN=$CN" + + truststorePrivateKey="$truststoreWorkDir/ca-key" + + echo + echo "Two files were created:" + echo " - $truststoreWorkDir/ca-key -- the private key used later to" + echo " sign certificates" + echo " - $truststoreWorkDir/ca-cert -- the certificate that will be" + echo " stored in the trust store in a moment and serve as the certificate" + echo " authority (CA). Once this certificate has been stored in the trust" + echo " store, it will be deleted. It can be retrieved from the trust store via:" + echo " $ keytool -keystore -export -alias CARoot -rfc" + + echo + echo "Now the trust store will be generated from the certificate." + echo + + keytool -keystore $truststoreWorkDir/$defaultTruststoreName \ + -alias CARoot -import -file $truststoreWorkDir/ca-cert \ + -noprompt -dname "C=$country, ST=$state, L=$location, O=$OU, CN=$CN" -keypass $password -storepass $password + + truststoreFile="$truststoreWorkDir/$defaultTruststoreName" + + echo + echo "$truststoreWorkDir/$defaultTruststoreName was created." + + # don't need the cert because it's in the trust store. + rm $truststoreWorkDir/$caKey + + echo + echo "Continuing with:" + echo " - trust store file: $truststoreFile" + echo " - trust store private key: $truststorePrivateKey" + +else + truststorePrivateKey="$truststoreWorkDir/ca-key" + truststoreFile="$truststoreWorkDir/$defaultTruststoreName" +fi + +mkdir $keystoreWorkDir + +for kafkaHost in kafka1 kafka2 client localhost; do + echo + echo "Now, a keystore will be generated. Each broker and logical client needs its own" + echo "keystore. This script will create only one keystore. Run this script multiple" + echo "times for multiple keystores." + echo + echo " NOTE: currently in Kafka, the Common Name (CN) does not need to be the FQDN of" + echo " this host. However, at some point, this may change. As such, make the CN" + echo " the FQDN. Some operating systems call the CN prompt 'first / last name'" + + # To learn more about CNs and FQDNs, read: + # https://docs.oracle.com/javase/7/docs/api/javax/net/ssl/X509ExtendedTrustManager.html + + keystoreFileName="$kafkaHost.server.keystore.jks" + + keytool -keystore $keystoreWorkDir/"$keystoreFileName" \ + -alias localhost -validity $validity -genkey -keyalg RSA \ + -noprompt -dname "C=$country, ST=$state, L=$location, O=$OU, CN=$kafkaHost" \ + -keypass $password -storepass $password + + echo + echo "'$keystoreWorkDir/$keystoreFileName' now contains a key pair and a" + echo "self-signed certificate. Again, this keystore can only be used for one broker or" + echo "one logical client. Other brokers or clients need to generate their own keystores." + + echo + echo "Fetching the certificate from the trust store and storing in $caKey." + echo + + keytool -keystore $truststoreFile -export -alias CARoot -rfc -file $caKey -keypass $password -storepass $password + + echo + echo "Now a certificate signing request will be made to the keystore." + echo + keytool -keystore $keystoreWorkDir/"$keystoreFileName" -alias localhost \ + -certreq -file $keystoreSignInRequest -keypass $password -storepass $password + + echo + echo "Now the trust store's private key (CA) will sign the keystore's certificate." + echo + openssl x509 -req -CA $caKey -CAkey $truststorePrivateKey \ + -in $keystoreSignInRequest -out $keystoreSignedCert \ + -days $validity -CAcreateserial -sha256 + # creates $keystoreSignRequestSrl which is never used or needed. + + echo + echo "Now the CA will be imported into the keystore." + echo + keytool -keystore $keystoreWorkDir/"$keystoreFileName" -alias CARoot \ + -import -file $caKey -keypass $password -storepass $password -noprompt + rm $caKey # delete the trust store cert because it's stored in the trust store. + + echo + echo "Now the keystore's signed certificate will be imported back into the keystore." + echo + keytool -keystore $keystoreWorkDir/"$keystoreFileName" -alias localhost -import \ + -file $keystoreSignedCert -keypass $password -storepass $password + + echo + echo "All done!" + echo + echo "Deleting intermediate files. They are:" + echo " - '$keystoreSignRequestSrl': CA serial number" + echo " - '$keystoreSignInRequest': the keystore's certificate signing request" + echo " (that was fulfilled)" + echo " - '$keystoreSignedCert': the keystore's certificate, signed by the CA, and stored back" + echo " into the keystore" + + rm $keystoreSignRequestSrl + rm $keystoreSignInRequest + rm $keystoreSignedCert +done + + diff --git a/kafka_consumer/tests/docker/ssl/keystore/client.server.keystore.jks b/kafka_consumer/tests/docker/ssl/keystore/client.server.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..0288c2089a90ef430e795e93051f480bfddcb77a GIT binary patch literal 5070 zcma)ARZtv?l4Teif=h4@7RcZd+zIZ%gF}Kl1c$*Hd;){JYtW#<-Q9x|bgd{JRJ)L%rzvvWb_|0J7C&$Br!Sujzdle`LWiw&=j`Ng53_mz)ZD5A9VA zut7EHnkp_srC~UE)NI^=Y9dyF9$YoELr=lI%RwE|T{oRFvzh2A z*5H`@23oWSjDcA&6^YCU$giAl5?o|b6i{)RPm4Pg{eGJ>opuYSuoD9|J^b-(&eqeL zXJ11&3{TYoj`2qDWTBV|5oa6tEd-RUF8X&x>9FOmh0HTx5+tHwKM$@I8<%7HKH}LP!k6OfEL^6)#x@OuOg+W zIXEw8uh?KRTXe+lRwr;rd!(Eop!T`vhVsQz(5dy5bH0%)*1;!aPpb0eYsTa{OfEH@_8MS2Ss?lt01z!|5#?7+3t zYSM6p#B!J43+&tSIDZ%x;M*n#o9OQXJ?}LcCzRd!)vR+WVwDSG-Bn;o>0`~kR^5>l zyHLKAZMiB-CVk{*%ybJ&63HykAI3vKpQOnDO(Fj(>q#;PCCsO4W~8ETSy) z%P@%;Y>8HRrL0|@ZLNN9J&{sZz601#4Y)B`L~Q|z_IlVn+FhF7j9+eV1>I0(!suOn z*j*jfFEPKqnHUWh*WNkTydUBwBNWcaE5nA}6qIbcKH>goff_$)f3NM$8Haf>QwIVcjnNVJ zJ0?<@jtcbj>r*>tf9KO!+xPI{HUBQXul+mhv)1aqXz|wQV=WH;hgK1R9Cq<9((zUK z4yhs6h9hx3>iw;K?bph-#Ak;gRVig%Ux-?|KYO)NvD9w`(4=p_b?Z>m(N*k>qHLhv zZhQ>%q#!$SfLh0s3Mp|jzgimcQhtISv&0X`{Jp45M;2$740F8 zz#ZD~vjVBDqZam#gSK)wpuHW~VsAvkgcuE-J3e`j^m-z9tGa0cb?~NRvAn{}%RCu7 zulWlrU8JT~PO1QZ^(lV2QvS^M5_c}iQ3u|}3n~9JYi-v354Pf+s<+fBPOEYc#9OxP z3f*a|3}BuN>EglLrrnPhxN)AtNSNWjWN%zNG&MBGTIv#Nt*=sNXy@^PA1M!SDTNst zhSUAnEP^o;xi{=Ek-%mUiDh*oeg0@t$%>_BQvxSs>I$waWglc#o>G|P6_iC=?-SXn zc2B^^aI2Xyw&((+&av(>#JmxQO=2X`C|fzqKj$8!VbmD;WrC9TjZ0tBH}E~?ZX=4- zKR%a3FkQV%dE}kcB%o)G-{Ykx^PS|&e&e%v6OLp*T_8oUKVq=b4}6F~U}?<2?fmjW zGN)a-T1|*ZY z&jB=XvCr4Y8#S(V%Di!Oqv(OYT0x#o0pAN>ql#Y_Dw*etsX5J0y8Jlc)%+$d@PsJw z70q%57*kLF?Yc^c{J8y5X!f#E;Yhvufg@7k;*Jk|dH+0Ch;{cgQ5UpUepHr4mx-N) zsq4Hk1Q)CvL=M=_#-p-m+SE%@5tcBRwaKx-MEMN-#E5lz0=t|iWC6Rmw1+BD4H@9r z=9$t8;poZs=d?$=zk`2WFiO5zXoF+RS`$BbbAFe~-WjyhEW_>E+$3~kB6_XFFuN!t@RdTXH|G2`it&0Y6648qIk*{dEaLPfN z8EVKW3!?o;ZqW%j390ZA><}Cg%n?lgdYiwID}vkqZ6V?$M8emywYQ|<xqnpv|4!^JL`WKH-IO6C4rtpgIjh>Q?56eq z6S4Q*cDCZQ``naKzTooooD2&L#+X#y{2KjsjR?&q6sg`42ZMk_81^Q?8#Nf95elo{ zq%Pro#r%bT(b%qp(W+cQST`B)WiAvrc$Ox*5-n&eX^5TpV$It#%dSeam^8QeR**-% zZ)cS@N-Y2{YJO-+06JQ1ZgWBGnqW$i#X!^pyHLsuP5Hi(T0O}yXjuFcy;HO2soQNZ zZ4EL_`H@JIhqgE9bZ;d@cDXm+>R~Y`5%6~ME#^!sA=~qd*;tdxEW(Vv{^S8cfzmcK zQPgqn^-vT^lku@-OK@VWMm~^%WJ{DX(;%@M8JA(C7E@BJZ?dSa%x$S6GnR9Yda;jN zpO7=AGSxD94r>S1j-lKjytR!gzg>*|<_(Pbo|h?&n|1(y_-gFIzI zqPK-Dsi#~_`|GoXhIZE@+tdYmMYoMU+>prDR%D4Z@8~W<&BhzYromL`AuHXGMv{X& z9EP62>CP{jk^eIb>f}d&sr5UcN~l*R9a&C9!dnFgDE%)jIsRYP_BYP_}go4)1hp`h_&VI%O=m|?JY5aN-qlxkjVy4TBtBaA1Zk|>%?}}LKm-uDna`dbD z@UMvRT{>;tYF-2K3%vCG-qJt5rVEbTxM11fbSxR9Gp;oMJ$f=+=;RYA47lUv_wT?i z60u!VljBb!zkuJ&iEdX2!me}7Ot@8-eYj@t>yC@99s8Rigbb?n6t^db10PQ!%*$Ms zHW!;Cf1&0I61+Q7%7?WH6w=Dk6J@aJ0uUmkFKFR+U$|YSHgP5^aJdKTQKypqX1y&x zJ?LH@$HA0!r-mTwWLFjxRK4i>$)bg7cv5A{^S&fH6i>h4dg7i6RytLyVK9y5OQB`FbGaopFiAoDTJA;jS*f7@J7sQGml12H1drauA`Xd6Z{+ zb6{8uGKUZ?@-25l^keaARcEYGlnS$QR1(NYpL4vy{R+-6Y?d^D8|k*WY6wkzFBwid z3tV zf}{IN4y@|r2^HcUM;rCm%C#V#p*wrg>@kyg|4Vn$p+9+GLChokB(d8$FdJh^d5PbY_zj@c$SbcuXmMX7-9{|)tnn9)a`*%9_ue0B^D&(SjwnPTv=vtpISiCDwIo9Z zVjVBbQn5HTBWZtzDz}!45fwYBbN2k(MVkAv@ zEj~CZ-gL{k5oeR|sfeu7(rEUcF1h#mczIir9EN7-Zv=Lk{W;qyUEO$|vU>){W`iJ!wRdPmQypame_O2>E|ACR4qq!f{d~P;hkhaz&4c)c)Eh7U$D$RF=-35uL#tH`Ek(066Gh;Mm*KOKI3x$oxr zEi5-l*=~+xFU%#Tb$F6Q`J#6t99WZ`Mb4MK8)h|+8#ZWxq%c94cQ2(> zCFTv~zOTNFM)T$|`4hn-d`CrVDP%o{nA{3~k?dEut19D$1#&1jtxvb~m`yuZ&$fKy zGu0VNOL8<|RWUwHYPZEC5^{s!)*2rc^r$dZWi+o<=wG-tSipBcjzk9(_0nzRtM$Cwm$#O-f<0;AF zshwfNlci7B8KZNgD|l&ot23wAsT&~Hdcl%m@0Lj;%4m=>h!gbcpC1<@02u*^27ph7 zJqO;#rf>8`Cz4@R8PBhASGNU=#T!m2m>)AbBN1vM2fl%ixDt1a*`c3Kba0qH7ytMF G?7sj#bCNRv literal 0 HcmV?d00001 diff --git a/kafka_consumer/tests/docker/ssl/keystore/kafka1.server.keystore.jks b/kafka_consumer/tests/docker/ssl/keystore/kafka1.server.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..e23d801fb2212809add445278cd8b1b2632ded11 GIT binary patch literal 5070 zcma)AXEYoPvtF!Tq7$M;lwHE^TAe6C5WPhgM9=EIXLZqg3ybJv)#yZvAVlvgdi1Ok zHRR@=d(Zpb`|JCD%$b>I=9x49o;d?S(hTF`;6acyo&?0;Nae^2avVY&6q3dUh@`Rn z3!6cZglGS+M7RY+60ZG)F@J4=fb@S^WF$DaC?sL%Uoa42`ws)39AX3!`&UW^;RDj5 zTx)LH^>KAUn`T(;im?@}yMG!kE)OLx4m*U5farfF;u7NnAS?vLFC&$4Z18|MU?Am8 zeMa)SkINxF_cH~sL-Uiri%`yjQkNxj&)*<38)cRKpmA2{eOGiWD$)s@1t6{a79IR;QMDEbsPD}GeL(|E=n&#~w<3*~O znl@9uUk7RGs$)w`%i+*VoEs+W%TF}|hbm<1ofUJS@vk3>2*lL;(4wCBOSeHoz{<}P z_PjLC60d`L&{RINTZEecCfk4^Ih;D?f-x^j$^BRw6h10&^)z(pkxCqaciQFaW>3E0 z)JOH>u!-?F?Z{LF2``q7j%5(vPf5{XN_@>k-K>|wlb7JkIx82fPCu>Vh&y05x^;bo z)=~@iD{ssz+5Zzb*sAp-;9emLnu;~Eh=zAN6)|^i%JaKJwFGf*wp%ZQ z>Tq;)a;+#NmDvE@g>!!(JmQ$fL_X<@sStWM{&jo$a(uX+Pdsl&UBRrlZ;eE`4YP{c zUaoP&OG9wH|L-XPaY$nG^O zR?h~26i7A22rml1CUtxlN3fgd6F(O~8^f~TXlf*=z@hVnyRI)uUPd7rpf8A&=KG8h zVI6srHI}i1TTAyCDt4vVOFcGQmSF3=p?|zDNW`xZ;cJ5T2=0SjYd$={G8P8*|5BxV zi}7V@_->EfTrJpzs}f3P|4*N^d5C567`0zmqP1b_#s|p- zADHk<_;=|qED!uryQdxU9jf~>ni#A_T>QExUNniB)s?{)g3AGq(86sx>O%RB>dvg6 zj-2i5m#ILWIsq1ryblyAiUz+41TWOm{E%StV!W%*{BHR}`sz!z0vYt-Ij!{Rbu5$# zFL|YexSj=#D_54^`1uuZS3pMB_}e5mWe94RtAdgGkDWnS8@B8EWB)4j*otL z_wb#$=sC$7>27xre1A%7f1gm^MsuGrac^OuA_m`CjFtl5eeq*=y=tIZF8lJq8l1R~ zX?~F?paEkxB^d7ggVl@s83pR@XgZiHXSe|;BX=7w zGMJI)RLMiwa8$ zsv=t3tyeFy_Bl*j-|2^7?5&K@=cuM&UDDSptBol)S<1q!O)Dvh&4E0wqlB|8_^%6o`s$XkPj*_px3(b&_t8miu{x>J^GEf2-dzfe5c#ph+usa{Uz}dst2>4 zoSjs=|B|g#x_7XKH*53}QQiva7cU0(TEq6YCG)WYbA zaO;Z=R|sj`uiT}!J9>MLojry#k@~|TFh*^PBB2;WL?IZ+&aT$dS^=)WeAehRja(Ry z(`<_Ys>*RW;#nqJXSVkcLlN7NGE3epVHm9&le5?r(zM1>TO6E_hdAFpfp-`&=LTP! z4U`Nzwpr_`$zLd#cQ%*M3?l%VrQQL8^n2uKc}NSCmuwR<%UDQY&~+3IiFlD%IZC|X zp50|3l;p1{Pw}yyS&%KnRcJ|+^ix}T;SFsisq@w=M7w6 z*HuOq<}FmksKFPudPNeef)QPogkm}mhEH4(gCqwIDPGdvF@a)_Q`POH$7~Miz6l?1 znkD&Emv7XwIfc$JRxq7J66!f;+}5Ux*hISB5}f4-9>+tfjYJ2{6@MvAXo-V4$>Dn3 zT$mB@%d#Qvt*-r>x-XBxZu^U=WKqPa5-jl&oW9|ID#$~CIN|B0|@fan0 zFz)a`^6dR=SdP^jLG#sUeXqQSlIMbU`*x+NDR@`WUV*VEjEzH^dpIA1OLisw=>6GY zw?wsc4!%;n>1#fh@zu_l9^#}ER6m>~a8qR?uM)Rvop;XUiAQ2Ji@8c22rmJcE2E!8 zb35QnCsjkS^}>#djNo*VXw%|DWRleI52H0JU!Ywh@*!W(uVgC8-}qiPFqj?D^YT~b zZ~}jignhyLg$kW9xuN8d$+OnhT(g?4E6|q$y}XmJPo-j! z1l!&&){1zln%IX9Z_iKMo3egu48+NolPLaD7Uo!oCar$t$aF8vGQai&Y@!%suD-F7dvqoTjTe1k;hFj^%+m%ZynF{2X0tjqwVjy3*}7D z{f#PnpYe^_dtX4`>qzY%yUqMq?8kLZ-yP}(%ItA}N8|r_f0C$gQsG%E_lI>{<1XOc zoJ;@rWm!PguV}Mo_~$%a_kAnRBQ~*JJg##ObGq1_k z_Fz@p>_XOt2YGhZmLE!n-NCk5wgKSePPPDr`CkXV;w{OakAI<;iY`F>4f`9}Bb`*^ z?~eH!Miu%GhnPY~fZ6&pXP8;=N_QoMJm~y6tyn?Wd(cGGl!o)L?zFEH$?pK^B&TjA zPr*%`^{#Tav%^u;PwxYc_BBL0e_mxuId=)F#A3VH{f|-UF(o`gq)iQS$d0#P7k0^= z@oF?REUR^+>ch~r+_c82+K(Ip!DV)z(7K2M_*f3y)*W%Y>QyNJ@* z{zp?PO}d3G^7PVhcZWPjVg_Pgj4hkMv2S?ds2608jf>XionhMEWs-;MPd9k<;8F^P zt-xe=nY2BbR{hIGZAx>ldWr)6&>3*@jZBa|)otSu)y|S$8vf#|aWSvO0ycGK-HBu= zKr|V%Bhv(nV~2bd0a@lW*XnbF;5S3MmA%;q6Wx-md`;7YLknjHLuw7(p6B%Hh;q9E zVg!-ah1qE7NhPvkdYt0X+;7iUFn?5$nBvx=WHFpoTiH zh7B?Zuuog&KDl{?z}2>E@p?F|aEc+Ud0RFL-@^r(6zmt%Xnvp_x9!7+rY-?Kv}ENAAkSa{q|<@j@>57T_o?ar41Y=+{n zE>S*(jH(M$WSF zmu;gH$;5d{%v@dOQv1vug5op9-o&?fR3HRT#q|5tU_<8TK5s#80;BZ9>=)UI78`c` zXh%EA#ZF-!{yi3h$tDk@F`b`9VnSGiHqt8hsiYyw4MqM+Z97@0NulgP*8WZJd81fpc;D z)qX`XvegMD?WbQB95We8mCQ;-ZlHShjW3rI@AdF$PIhFZr$5*7Iix6_@FRX-z5Y4u z`uBo4pSNpQT$E}3fz;WY5}1WNu%*EATjZm+cvEp*60m9&ex+EG;Jdr6AAR2L(V^sQ>}C;ld;pe@xIx1~7#{V4=O-GStf|8p!CA#n#x3?rsJNkt>))Hp|Hj~{UES@~!(=gi;j;T% z39eml<(RD|s$Jf3sk_hq0pqMFE%__{OJk70Jg05@| zQb)8Ifu~=)DU3Wrb^h_sCocb<7pw&T)t4lRs>EGlsp} zs6J`qM?^T~Sl+J=c<19)moG(x^~|8NcAuNIs^!;+ z_*GfzFB7Y4l*jDxhP(+~ScTwt`O!OJQ!)S>E@{isG0ub@XfoaWZs_uhhkO+^UG%Y| zR3IQNMOb3#zF~fj_Y5&E;huNGsbu#vGdm6Hph>UK%r%wiEm7GJbqEwf{?CsK7Y82z zWC!U!&9C-L6DoRe4IIrZ>zCSd8mZhOloxB@CDMkI+~Tg+#9JEy@sIPu4j;d L>l?rS_euR<^%Rj| literal 0 HcmV?d00001 diff --git a/kafka_consumer/tests/docker/ssl/keystore/kafka2.server.keystore.jks b/kafka_consumer/tests/docker/ssl/keystore/kafka2.server.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..140d324fe245d5471a1768e6eb32decf439db1cb GIT binary patch literal 5070 zcma)ARa6@clTAnn?k>R#6e)qypdaoOFVYk##fwuMiljIc_oBs$6?Y9@+@-iXG!%!z zcF*ql|J|qEhdDEIXYQPNyK@GHBpbv4f?!B8R|pPQh*HQE0T2tAgCw&6Bgstv!bUJ8 z*7^S`VQqtvSnGe`<-fK7!TWC(J}wX=2ZT00DBD+{&$KG;|7!eE(9nq zaNIt*{$NX;t4|LpzmXZoz~CUp05ZYwA=v*l5d#Mk0E0qs-i9avEkIx(7noT8{viGh z1Ra;lqax8k;;sF65n|^Slw;hB9eUhS#kQ5Uw>-Ny(GXq(F{FrQh+>-pk3%{wgTobAcMf=+L*G-`k(ofD&$y`nAO|2D+h7yj$Z;WWNcYVRmgHw_BR8>MpzRrq(}z7I*J`3nqBCb>O;K?+zI`y_-4gD5O)$wP( z7gf(0C0glGSW5fNRrF4jOCJh@Vd5TuG4gGf!lYmQ(rhZ$ad9hKYOST6R}`Xih)g^TK5@?JTMnG@STRU^Hr_OPw9eH{UwS@zJx4Q=v#u)5X}$azk%Wygycrl% z;+u@3Kef{>=jo~L2c}I)I$6G|EGa?nr=^9=@@3Wbk2GzOO;X$A;`=t+v$XUGc;wM$ zLu$Lre&XY2Q!sjDW{eZ80wZWqD{6J|I~_~{ADTG0rd^aob}s_MCZ*NeCJKf*_QeF7 zUQ4{QVIcVc>+sLyC(98p^4zdJbqzck2xW4UQPAMd(@uP8F#NK>NmdSy4nd_CY&}!a zXgIM5JctV1VFB|Y`=Rp_?95@nswLSJhQ=GS6yd2Tmy!LnkjY}omAVr*`3T)T zVx{FdhJr9d#i$_q@DqhOs#NAJ=if z;1P$tMhi`+^7aKr-*Zf@fQkbWuUW7#0RGE;e%QMszsi9bn>f4sN5K|b)Hs=xLUKdX zSBOJndrA^|kkk1?ystrmBOVLbTRB^4pk;gRRfG(NjG-lK5LYzkQ{`U*q4{$9jj$^M z(FF|OZ*<20d>k1Jmo)?hccbKtu$}ci)nl&orfNUqm(RH;JJlsdZ_(cpezj{3FqH@Y zuoQvCV{2bXy^$)2T6^flSGwo#jd6PPzl-#DCLBNiJ%EzPrmROl?bO7Zr|;-80U}6a zG`^4mWlP34#ctX;aK_vIPO{Q#N+Rxk2cJDUprd{53uRUINV$X%M(Q zn7*TrM9T^kyD*DOp&H^k@&a)@#WP>f!07XPRE~|)=wDM=X zggvzA;}?xLV7#i{Ii}-MoZ$0vWz1W*cAgTVj9Sv}czpg8l@(L=~!`A|MQh~+WA=`4_eI9(@0W{0N!sIa$yK* z=s2wRKwfgCJX_+foX1TmT>pfOi>qObge1Isuivxu2#`gsl+A=cwcrq2ELF78_jOkT zwAla3GJ4;Ng(yj(0XOW`j{j{L{vkf=`~~=Z$kU;;KW*(Jo>-yugzUdOM^U_Zi~df@ z%Tfwvhl&*=2}V7Q%^jXTKnKUy#o5d&H-5l*b9L{bUZ8Ms%#&!Lc>(TiL26WydaS?T zS+*W7L()V|_Ykk|Y)6?Nk=M3b=z%*3m19Ta>Yg!Qf?lW!Co(f;b{95N;kULILnrN~ z5XSlRU<>mG0qP{|sr+7xU4f(l8a9ihI^QL24<5UM3cvXh9UVz(87?GeRg^TVu`lZA zYzk2sX0{uFNQEESpp|$QpZ2O}bacNW4ZVCy-vMN5e%*BdTcn?WcBK$2t6pvHtjWwa zQx!8&VMklnG$)(xrRCsLqhjglw)SAV&8v7P`1MTuA)bbIj+wrG0MG$8k>d>a*V1}`A*$mJ)^Nm@71BT zl8eBX&IL7Gzaza2YN3pfNtP4AO2&D5_ZtYHyIb$4=kSnzw6OHhGM#r}Iiw9uN&5At z7`?QT687T#qQ()8yxY<`HfN6!=Fc5)_D1eey*kWzSN#13UzZ>JY5`C@0S8~QF^FYF6yzR>OV#W{HvgdID_u`qFW zuBvPmD?O3{eiAc6s<|q`mP3Mc#9`vF#!bORb~6)Ke`n9KlbdDz?94VAuAK@wANJ;W z{HO=~U=I5UYkY1D6le4nhmgkG*PjHnm{%Y@X>IOp35-N?I_6EDtisKdYrhM(Ds~S? zto2v7P7UsFMC!ylRr$YdnxImsDHR^XQHHatXC7& zMlE{=61C(BY z6^_pn^F&$m+9#}*qvM(~%aR&BQq7z{pG*~>$(1_YX^YmU268fxGB0NVzB}Slvy!8i2JLF z1P*4rd+tqNc)Oj`#;aH;c{V+Vu9TI+Bs44E_AIIAgpy3zTKdWoNLBD@eTVGbF2DGZ zmiYe6Z3EAiEIN{R*Wd~cNThwdVc=bHzX)9n-s8R~7N8*CZdUV2`(Ct?+CrpP%36YJ zK4Nx#*Zkw~fkDpJl5{xjcX$pE(V1Sm;^8iT#ctK%fV#I_(_l8K6eY4ye?QGBQ~5_! z{_wJ>(uet*P!jFN7!4?El9yjuMFCanM|G5kyNkhCH3Ejx<-;{%w5DdNjk5?UE`s?z04Dx}^Q)Mb6_8&UGlaE!Rh zbjc8QIwxIZ{!?bFBiLBLcwXGdV;1}s+@BbObxIom(hz*+_^FD^zP*d4v4{G&Urc=c ze&5^O#r7j7Bi0NA;Tr0g>a2Nswpn5ppgELxMOU6d^#}`02&U9Uu-8-@ja7=>F_!D! zmVW2y5E{D*@1|24DgZQ8uzR@0!jlvkvbHkiP&rJ(s+ubPpy4@FsK}bCoC!4jgR$Zk zJ~8EJW3O;PXptf-8Q;rsGQJbt(4Zzg-aHaifbznJwj0icAT?H6xR)3MG~<)us8_^= zG|^}aDwF#C=EG$L#3Ir_l_t+Zz}k>09iP4Cr5+A<@#(F}q$S^Jb4Th%nB-zME}zim zarj(!FtK=iriqT3puV%Ty+&Pp`4`9dhb<_P-4j+dFBNYq3gi8Sh>16l(t)6^KlsLZ9bV3V8st-Ed3=VsK`ZrWn}0&nelUW zmO$(7zUt=VdNKL3GbVtq$yw-m`FMo;LxEpL{G7rBN0rNjq4LxdY0T01gC|areJCU2 z(RYTlp8~5*Z=Bbw#J+7$uPknnB|Y<<+EaQn#_j|+HTTx_Nol*RK&AePpg1+e6CwmT5XXo3XriRGc>`&-qsd2pnE%Y^0 zBAU@kL3Dk2>(eBTbY!#3uwtr+yd6DLY#=m0AYGLQYF|~v-BfxU3+L_EX?c6#@H2cT zKKD<+lMfIhN|_!jl_dS+XbHc$zMZcdp*DxkLu-9Ry)mb&pO77}&I5$qV@#1pUN^cg zBC@FQ!D^%HS&-y$sn?VuF|V@zU<+^;SI$<-av18%N>r==E?k6o_9oy7%nMS%y$M=} zy}T?cmZj?AFXtz-)j5KgrBxwI!jk*+KgBP<$T)8Fg1fwzAv#Cj?Hepu-P8EWa2uaIy9f@De|ZDd*~|=oGk4c7`VuzD0eUPkw!l z5RpuOmuV&{GsM|FG#2$hv$GLIOLNg<%BW-&bi7O(^XOnqWxC|7^0uVO{a-}0R2bE+85UtmecMxNXJ1MIJLUE}%OI|na~AtKhU75bVu!y}IMoR1Pf zQp+DH588iuc7a$znaPzsFtlo1u~)O4>@c@8Ufc0|DfefSGP#!Yp^6X-#VT6R@|Tli zSfhhyP33i!e>TL)$Z=Geg+Oc5Q1T9Fn$=pzkXa8KuiFb>1`mU>{ZZv z){M_MUg7}m9ET6~UuaMNOg(wVr4xq9#|S2$bNmTbS1=n)E1XPRrRA4@ZoSC+pC|Qy E0Nx;QmH+?% literal 0 HcmV?d00001 diff --git a/kafka_consumer/tests/docker/ssl/keystore/localhost.server.keystore.jks b/kafka_consumer/tests/docker/ssl/keystore/localhost.server.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..2d0e7b791c41cf80943f409aef3600d3b233a139 GIT binary patch literal 5086 zcma)AWmFUlvxX&>2I=mWWtUt8BqT(T?vP%(L%O@WL!^~PN^LWa&nM>pl0L z_q+Gk_x+eNGtb1Cf6vT7LP=HtsA!N-l5k9H?ns5m8+=qCYEdYOCweG}>tEOr5(>Qh ze<9!=dMFU_7heA>%b2+TTZM;%3MdK%2LA>9Ay)rXVBkaaAtL{B@gUF9NhcB`#RUW( zAQG@uA(UqdmYrSBRsaA85df7Df`^IqKOF(s7$^`rOl;Fg1ymRsIx06hk&XjgY)DJj zpC*sl%jA+^?e8E&#RqrYUVM2*obn74Ms82IKA(+Ui>?SZl8gtg8qG-gjL9S$Kf7wj zmnl4_zH}92uFHqgu_|50#8h~h{ch?O`;a6&tcST-#K0SVWU}0NB3*cdP&i*XUVSIL zPy8jaVaPefRijL9*qv3Rdc zMX?D;U8w5Q7Ef}}Q{2=*+8u#1TXh*T+h)Ix7qhw(Ey;g(M%?jze+|c{%=HN@_^Mfp zJh`Nv4Q5hQP$Hdu#T4U$4L8uF$rwdBHshHd$2ScN?vhmQEe{~`Zt-Xt#=&ZzrWL|u zRUbhTwL6dWZCCi2JL}r0;F7jb+leCRItD5x7V>^?o~Jjdro>Kz2v>3L8+K)!7#XsfE-)2z#+)>yaXBuzY$L8zmBNz!9i);9-oIFlQh|W9*5En2I~(+8K}o{%$uP6pjqimT zo~wXKx>l*}Vl^Z~168d{PX+-pBJo{Id9~vEW^TMHO0Rd8_7mS=G&S;5`FHB~h3}KU z>@aQLPR#4A({s3O))-1Z->Y*cTij+F8a8D4vrRyKyjt-s(E){hIwn(h%6h0)xy@3d z9!FnOmik)tQ)GKCdsQfUNtR`SGh6NF>ix7i(wWn@46Pa)9`_cl==X&A2Lg(HYT%(R zE^Bgk0k;zc^GQ1UG;(^KPl6`G;wwllY4{3_X{va3?0w;Zo<5|JL;PGgE6v8?F@kIK z&D}JU@6DKd?z9GUv_Qa*qMMCBZfvK;>DLoxf+O5e@H$wWxgb_OM7C|qrXr|50(_B$ zJ$oqhqqzIB>y%xcbQgK+-0e+59V5jXpuiW3^Pn3ZaW#pu*=X(nk#Sm9YI>uui>xX! z(P+cnfGmY#vHI42Xyej6&3A@b=2cEc08 zEJkAzd3iPC4@Fv4JIMIs+D5QfMZbRT=axHuE(HAW2HOo5%-}c>@w`PlR?K}8>-jz2 z5iAz2N4`+~+|cA&MwB@tzI~gqyv)xp!;5$n_ zDIPlLg<7qZ*EgWrDX!864^O}2GlqT(=d!E%3HXc{`I}KLM&0#p=)_&`MMLDXv&`I5 zALkQfezu4koSu4WH&^a)mQGKqx3_iy6MVjyamAdyYlb%yP+m%9XI}DPb9ov&JNM^u zH{#2e-x*7jONqK|hDBt1Uul@Z6exd6sM1kv#3QC%k8F(IfFVIw2PFY2)-ASrw|!o+Z<`+mv*V{gH1!4m?%91o*l5gutf^H*%WMe6MrmDH=pDse4U*XU&-`>W-n& z8rUpy2>?Y0hq)aPI@6Cf@Nk0F3&E|yY604P8i6?_L4-Ol1tl)9V&AryjOVuqX-KJ(94@p|w*t8$hGMx~7 z@(`wf=oXulkCdJm#R0_`#RA3bueSMXxubae-xM-FQgmW1TSpippDzL@XdfQoH&76S5zD zd7Lh?@=Z^*pL!^W#<386IV%ygD|yYqETNK_pi|vyz-3RB zlsk*iiBWOnzw`zk1!5E%3t9n(pcPM3oqi6{sVM>nW$KGf26ckVbY&y>m5E2wtA_0R zad{`rX;srSy<*0we*mfWH;|GDPKi@XC5Otec-{5)5?N307~cyym>`5iLMw>;9PbM@ z)5>rZb6b>qu+t}|>TK=K7zp^Is7c4$Ve(kt)_JgWq5+3`-|YR2w7*HseeA52gQUU*jPD-*|?RL=%F>#!@etDPa*@1TjP(oeNC~<>d~8wiQN-`-|+US z52Gg8^KPP(BE9-{1@9CVWJ=uZONO-0JgE@AY*{?M3#XzeO?yDp zh`r{^lP{IG^W7A6sJvjdXj@ySqnFSbm>3wPNKIc#XyaeI&+6%#%gz4kO7;ELzg}4< zPpPk5&6b^L%#fs(d){wO;Yshj8J68K*guTUvz^m)X)^KK(s~kuWeKcQeJ;`S%L|^> z{=#I*MM$z!pBE#W=EV(r5?ctj3*IA4WF8;Fo3vOlP(K?_^U&_*MiA3X`NUeC=!rV5 zec&Gp1U}U($Scqhvd=kpg4@~4JcVF8ha&eQA!9dBA3;&E7JQ7toc^C&g04hp`+rQ) ziDEJ*zQvmidHf5epJkVnZbdf06*qIRsi@$0TEwoo}YkLTb*ouFR%P%jH>hujneACu}EViF* z3ej|oq38FAHZW7z?{k(Ldhyt|)dj*rdp>;lIVN7c!4OoQFl6#o;$7xD9o<@;Nkjf&-NfgP^Q-fBuo&+aFQ0s%1kmL3{-46`3Et7rA(4m2$Ei5tc1rdtBlJhv zgF|iaDebW&QAQ{LTNZv+_Ee&QX&K%8k>k5wuM33a8so%eBpFIc=}~-ySCop%9@OxJ zAlK=#JDD-a>oA2}3%K30!M8GO+XOTb)lLo%Hp2%)$sFh5DWWSloO+uhXNKf0@7JHn z$p+O6pMmzU!{Tzz@DW{%97GCJq3NqKtoUI}c}^JzVY9b_$)NBbc`kl{^YSD6cM1U3 zcid%!;iWscEHaD&;%Z(Vi+u*q^6(0jrV%`u<0d&ibO|GKM@5|610jhvrUvRprcf(yD|RS&;gZA@NVu4kPbt9l-g zZlD%I+}6K3??uo%M1S)vfhZ`>ksAa%MD7oG{q&TmGL>tSA>!E0WpvMYYOorCS_tI; z3*bM!IpdtBAw6~UzUGWI=L|hDs7MR!d(9dC%R$B7p2B=zJ*#&>Ae>;5>{fT3GeQn$ zKj&nYmaWI_ZC>$Ukzbpe(IFxwe&mCSv5kPoYr{L2Kaw*eo#2S=Wn&Bi)W=bzr$&gP zL%jTjkm!n41)I};0*;nM?&-dK#uUTja+>x(ej=XmIp3opZ8&Qc36o$G=Y0$9q{X(ppGR%v<<%~1 z>yyoh7$oL7HUv*rGYKTVW{ec;A$Lwz^({`4pv!jM-JzJL9XDHhE-4`NhU2UK{8^7~ zE(GZs#dLtrF9jSGy7XfCg3Gv_u7NKyx5#|>#bM5gxS>)GEFJYT@XMAq`CJ_9hOW(a zI-k!sB%Q?%&O6j<)Jh~-G(qs~ZdaWLfbvQwYk1R{j^qT_X0ul0fu}j#HA=y@g?AVU zu|xabRIt0)xwtf+{~kLtPp9kGl_+2ue!QaRB3DzQ7GVyFUF#3TIbjUArxxAg6g?C4%Xdk4!&EhMAD@l z4>cq?45yI>Qs_^W=J;adn+FR(@`+~coq3uxQnpft)joGHfzd^ye;Z(^hNgvfWz8Im zX57i+{UAVLQ?T-ZgVg*B*s~MVgXynI%)98lMfQ}5@pjI8{q!`cQk|4MW9tW^Gp!8h ze#Y5a^IQ$IY50?ZYhTdmPZ zuE9NNHe3)9erhyv+g`cmL>s#9v|WHFDj@@rd$nv%`z--o;BXJ4rNg4*YxL~ww)e-9 z8_T19atBPQy+=i*fGii4xQf)VWiV*a10pS>gr0;f) zvsc`zN0`pUT{^w%puUo?9S>hq>964YO!I>k;=WYewG4Qv=LodQf){W4y}X@6ZH|yx zcz3JH#hX%s73F>2g?mSwbg6p1=nRu+4=CFT=CFxhvoECjSKf=&f@E z1=&?ZbMs6o`bB#qKYdY@D5t_A9Ves6t(7qu_syU>A*)>C(EMSEcPB_V(JX&bUH7eU zDXK<}n7RI0x#lBdN5BF=JZ6kb^^W!v_+Vj1q=X3+40Q>#F|Sv+2v{?6iq4Oo*6f{9 zRu?B{67Z&T!O|bDkc}}3-_yl|1_#`a6e`woHony(6Kd$+86a6C%Ag~kzk0=elt$n0 z$&#+1SG2JEgaYB{dibEot;~6KbjK5uoBU&IjE!!Ct+lJIp8>1r*_-;T@Putb6!Qqd z-)RRWCH=A(NwP`3rWbng0=rK2!dDCRAp}5r6t6!WhnX@qd8$TSbhCf#hlEn1QU*lR zUE`ZnZ9mQ%;=&$3o4->LD}~Rsx-#PAx*@KSUf&65xJan5?avAJQliRQU}+64J7}Kw zgX7Xw&jK;HB&+UpS_lvz59@J5bv zrL7MiNU>bxS{tnOzL;5U8#Y%SnaLpLs8EMkYEbAP`|&shR6z U7^M^~Ccknf)V-(q-z)Zi0Xu_!lK=n! literal 0 HcmV?d00001 diff --git a/kafka_consumer/tests/docker/ssl/truststore/ca-key b/kafka_consumer/tests/docker/ssl/truststore/ca-key new file mode 100644 index 0000000000000..bee4cb1e35b37 --- /dev/null +++ b/kafka_consumer/tests/docker/ssl/truststore/ca-key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDePrGbVuQDyu3x +X5mFeixzTOWMDfAeKVHoClOhoaN0z0EwzaqBDCZmUiv9+Nqcvgpg0o0ZrOf7eHDy +ab4lKYE/BzOOCPIR9fEm8jxcD6UPCgzstW/28hHhfT9C8Ae8UtC5gL2Q0ezaQ3TF +OrD5l3O0qRYVmgk4vKCn+qCVs37R5VpLrD3U+VfpRFoKmOsHVA9bNRwB+9hTgPiU +95KUHkdjFI2v0LFFjpLrrF0KTQu+RMlQQ9v3/CFyS4kaMY4k0rxAwBI9gO/9sXtT +xurl4hqXqECpVc/3UFJgJZUez4Fvpe32DzXa8xSMs+16OW7ZzgYJnrED/IwnuzVT +G2H1OEBXAgMBAAECggEBALWH6hSjPa3nZMHCv6hqKcKoLdtfSyGnyJJLVRJx8Ayq +eWo+o6Wy4e6z8eEKEq7y/5EkikJNnFMohsFLJBhAtaRg4niC0A9hn2+bbzgXQ1Bs +ejAi52iDQ522ayTm4LqvvnO3N6ACCIAjl06+aO4GCmmnrzVH0353fJaYvfBpZA5W +zeV6g/ffUr8/TPkbEaIi0XK/qszwjK61695WFSNtvVvbPPbTlv3X2zT0h915RvJ3 +NrmJ4bES9yWX18mJNIJJelFj7f3A1QtjuZXgtDEeC+hg1YEnK9xlDwLHeofSNrpk +PNEsl/HBQ7mZ/gIFMXw5/TAFmmFXViDpSCmfuT4wKCkCgYEA7/bUbTKVVRKY3+qE +CKi7K56R1M+V6AGZWrEtRq9pgF5JEA+TQDgcEUdmjhpNcK8jwdSLpqZix8fh1BIy +yJNH/hTJPqFvDCMKWkPV5nrnBa0axzuhUrAUZK8cWZIwlknVkEvDWj0KI7iiCNzs +SdUglHjQ7KI/idA2nQJ/KOe6Or0CgYEA7Ri7jetz8BlPhIKurb/JBnLdqYvW5KPv +4LTfHcBlGdcRldy5PIlKMkVq4aQVfMT299xbXTQAzxd/Jiiv1w9KvllsXRFv9Vi7 +3rg7r/fyYLpEaUnVsVD3ZryLbvFnV1Hkb6PLlROcPEsY6RX4t0SPyWgLoiyoCpmE +I+tmEaqS4qMCgYEA6NAxiUBqYf+JErYVI9GQxuEx26cLae0jPDpqEwpc8Y6IlGyI +EOZfVORRO82JFKrXplbwpt8HN41WLMr7Ol6qmnjQR1OYyc21XZ4zCUp0ma05LdNM +0UzEfGrotn5/ZVB9Q2I+Rm6yR7X6f17YYUnGMkaFbRB9VNgOJIcnmCg7JoECgYBa +qhuGcKQcC0G6FVMAxBb2LVSQMhZRxlLDZcbwFJNUp0ZlPUtkV8eUo7D0ag+Bb7xV +GeGiKXGpv31ytRpQrdtSINXsXO+oLW9iAqdoafM4g+IHNSHPNXv1wGG9Yfi3jmxB +OKxv7FjRQZJcB3fddQ1a4juYn2T+8Kd0KN8WIFkc6wKBgAZbsp2rHGnJJVeClODn +ZCvHLsh5Sr2qftj7j8DMVV4fpgx/ciuuI+ho5PrXhZd9gJfk1IG2Pv+zd7rCciHW +g4OodNZpvEXVIZHfRbDDNLRyI8F7guTxsOhHGgWeF1Gwu74gt7/r8YHDXLnuQZ0F +CHeFAJk56bMFw68fae4u7kxP +-----END PRIVATE KEY----- diff --git a/kafka_consumer/tests/docker/ssl/truststore/kafka.truststore.jks b/kafka_consumer/tests/docker/ssl/truststore/kafka.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..4688aa0792b5504c1a4c2690fb9f8d21d82c9339 GIT binary patch literal 1126 zcmV-s1eyCVf&^j$0Ru3C1PlfVDuzgg_YDCD0ic2d{RDyo`7nY5^)P}1@dgPhhDe6@ z4FLxRpn?P9FoFZ#0s#Opf&&LNQU+thDZTr0|Wso1Q5#t+~Y6exc17Md)${sH-X9Ly)wCi;JrsLTE zE1FoLlRgo&V4nscOi{M)Fr^*5HY+n6aA@B!fI!zvOE-x5_K;CaofJn^s+s5s4~MNf zI!o>~Om1+Jy+M36 z|B?7bRD#q520D9R;Ctga7)!(o8xG7?J9WO!VgHN;WUyw<$x(b8&K-A?T2RrivbeuK z5Jn4WLrtimSV;CwjvtB}k8RdhEZDqY;WEY#z(W&kkv>;03)YH>2%$!`u9TNM3U3!VYWueIhs|<61r?mJ`XL- z6uPaq$@TLWh*kYZ@|39f$Gzbpbz1nvn2ajeL0X$vNY+h*GDL{WcZ6>i#C{Dx?2!fy zYdn&1L*1`|Da1}ATprrGw<*Yu?EOyi0XYjbAw*)Gth3pG1BF!yO6fXh7__v7doK+j zqI3Q?>+--ibM#Ng>Q2C(s1yvYeiSw*pS~p@9JMBM5QU_9i97E#AWY;TK}{Y|EXb4`-lCI!x>>-j@t$u z^fwmC@^BeJliqfKivloBFflL<1_@w>NC9O71OfpC00bZaXyz!;-~|Fd_W0cygfeWb smC|1ZpF113M$S3U42S6i6slK|+ed}p8ds4mjZsQ}jhRE9B?1B`5KQ_1$^ZZW literal 0 HcmV?d00001 diff --git a/kafka_consumer/tests/python_client/test_unit.py b/kafka_consumer/tests/python_client/test_unit.py index 7fc3067f70847..456be9feab544 100644 --- a/kafka_consumer/tests/python_client/test_unit.py +++ b/kafka_consumer/tests/python_client/test_unit.py @@ -29,19 +29,6 @@ def test_gssapi(kafka_instance, dd_run_check, check): dd_run_check(check(instance)) -def test_tls_config_ok(check, kafka_instance_tls): - with mock.patch('datadog_checks.base.utils.tls.ssl') as ssl: - # mock TLS context - tls_context = mock.MagicMock() - ssl.SSLContext.return_value = tls_context - - kafka_consumer_check = check(kafka_instance_tls) - with mock.patch('datadog_checks.kafka_consumer.KafkaCheck.get_tls_context', return_value=tls_context): - assert kafka_consumer_check.client._tls_context == tls_context - assert kafka_consumer_check.client._tls_context.check_hostname is True - assert kafka_consumer_check.client._tls_context.tls_cert is not None - - @pytest.mark.parametrize( 'sasl_oauth_token_provider, expected_exception', [ diff --git a/kafka_consumer/tests/runners.py b/kafka_consumer/tests/runners.py index 988e459be6414..3e6c69eb1f8b5 100644 --- a/kafka_consumer/tests/runners.py +++ b/kafka_consumer/tests/runners.py @@ -7,15 +7,16 @@ from confluent_kafka import Consumer as KafkaConsumer from confluent_kafka import Producer as KafkaProducer -from .common import KAFKA_CONNECT_STR, PARTITIONS +from .common import PARTITIONS DEFAULT_SLEEP = 5 DEFAULT_TIMEOUT = 5 class StoppableThread(threading.Thread): - def __init__(self, sleep=DEFAULT_SLEEP, timeout=DEFAULT_TIMEOUT): + def __init__(self, instance, sleep=DEFAULT_SLEEP, timeout=DEFAULT_TIMEOUT): super(StoppableThread, self).__init__() + self.instance = instance self._shutdown_event = threading.Event() self._sleep = sleep self._timeout = timeout @@ -32,9 +33,24 @@ def __exit__(self, *args, **kwargs): class Producer(StoppableThread): def run(self): - producer = KafkaProducer({'bootstrap.servers': KAFKA_CONNECT_STR}) + config = { + "bootstrap.servers": self.instance['kafka_connect_str'], + "socket.timeout.ms": 1000, + } + + if self.instance.get('use_tls', False): + config.update( + { + "security.protocol": "ssl", + "ssl.ca.location": self.instance.get("tls_ca_cert"), + "ssl.certificate.location": self.instance.get("tls_cert"), + "ssl.key.location": self.instance.get("tls_private_key"), + "ssl.key.password": self.instance.get("tls_private_key_password"), + } + ) + + producer = KafkaProducer(config) - iteration = 0 while not self._shutdown_event.is_set(): for partition in PARTITIONS: try: @@ -51,23 +67,35 @@ def run(self): except Exception: pass - iteration += 1 time.sleep(1) class KConsumer(StoppableThread): - def __init__(self, topics, sleep=DEFAULT_SLEEP, timeout=DEFAULT_TIMEOUT): - super(KConsumer, self).__init__(sleep=sleep, timeout=timeout) - self.kafka_connect_str = KAFKA_CONNECT_STR + def __init__(self, instance, topics, sleep=DEFAULT_SLEEP, timeout=DEFAULT_TIMEOUT): + super(KConsumer, self).__init__(instance, sleep=sleep, timeout=timeout) self.topics = topics def run(self): - consumer = KafkaConsumer( - {'bootstrap.servers': self.kafka_connect_str, 'group.id': 'my_consumer', 'auto.offset.reset': 'earliest'} - ) + config = { + "bootstrap.servers": self.instance['kafka_connect_str'], + "socket.timeout.ms": 1000, + 'group.id': 'my_consumer', + 'auto.offset.reset': 'earliest', + } + + if self.instance.get('use_tls', False): + config.update( + { + "security.protocol": "ssl", + "ssl.ca.location": self.instance.get("tls_ca_cert"), + "ssl.certificate.location": self.instance.get("tls_cert"), + "ssl.key.location": self.instance.get("tls_private_key"), + "ssl.key.password": self.instance.get("tls_private_key_password"), + } + ) + + consumer = KafkaConsumer(config) consumer.subscribe(self.topics) - iteration = 0 while not self._shutdown_event.is_set(): - consumer.poll(timeout=500) - iteration += 1 + consumer.poll(timeout=1)