diff --git a/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_clusteroutputs.yaml b/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_clusteroutputs.yaml index 2ba82ea6d..ce80e0d47 100644 --- a/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_clusteroutputs.yaml +++ b/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_clusteroutputs.yaml @@ -3267,6 +3267,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: @@ -10519,6 +10706,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: diff --git a/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_outputs.yaml b/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_outputs.yaml index a384445bf..6fbaebfff 100644 --- a/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_outputs.yaml +++ b/charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_outputs.yaml @@ -3263,6 +3263,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: @@ -9789,6 +9976,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml index e579f2b27..0293ff064 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml @@ -3264,6 +3264,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: @@ -10516,6 +10703,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml index dff193ee1..5a845c924 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml @@ -3260,6 +3260,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: @@ -9786,6 +9973,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml index e579f2b27..0293ff064 100644 --- a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml @@ -3264,6 +3264,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: @@ -10516,6 +10703,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml index dff193ee1..5a845c924 100644 --- a/config/crd/bases/logging.banzaicloud.io_outputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_outputs.yaml @@ -3260,6 +3260,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: @@ -9786,6 +9973,193 @@ spec: type: object principal: type: string + rdkafka_options: + properties: + allow.auto.create.topics: + type: boolean + api.version.fallback.ms: + type: integer + api.version.request: + type: boolean + api.version.request.timeout.ms: + type: integer + background_event_cb: + type: string + bootstrap.servers: + type: string + broker.address.family: + type: string + broker.address.ttl: + type: integer + broker.version.fallback: + type: string + builtin.features: + type: string + client.id: + type: string + closesocket_cb: + type: string + connect_cb: + type: string + connections.max.idle.ms: + type: integer + debug: + type: string + default_topic_conf: + type: string + enable.random.seed: + type: boolean + enable.sasl.oauthbearer.unsecure.jwt: + type: boolean + enable.ssl.certificate.verification: + type: boolean + enabled_events: + type: integer + error_cb: + type: string + interceptors: + type: string + internal.termination.signal: + type: integer + log.connection.close: + type: boolean + log.queue: + type: boolean + log.thread.name: + type: boolean + log_cb: + type: string + log_level: + type: integer + max.in.flight: + type: integer + max.in.flight.requests.per.connection: + type: integer + message.copy.max.bytes: + type: integer + message.max.bytes: + type: integer + metadata.broker.list: + type: string + metadata.max.age.ms: + type: integer + oauthbearer_token_refresh_cb: + type: string + opaque: + type: string + open_cb: + type: string + plugin.library.paths: + type: string + receive.message.max.bytes: + type: integer + reconnect.backoff.max.ms: + type: integer + reconnect.backoff.ms: + type: integer + resolve_cb: + type: string + sasl.kerberos.keytab: + type: string + sasl.kerberos.kinit.cmd: + type: string + sasl.kerberos.min.time.before.relogin: + type: integer + sasl.kerberos.principal: + type: string + sasl.kerberos.service.name: + type: string + sasl.mechanisms: + type: string + sasl.oauthbearer.client.id: + type: string + sasl.oauthbearer.client.secret: + type: string + sasl.oauthbearer.config: + type: string + sasl.oauthbearer.extensions: + type: string + sasl.oauthbearer.method: + type: string + sasl.oauthbearer.scope: + type: string + sasl.oauthbearer.token.endpoint.url: + type: string + sasl.password: + type: string + sasl.username: + type: string + security.protocol: + type: string + socket.blocking.max.ms: + type: integer + socket.connection.setup.timeout.ms: + type: integer + socket.keepalive.enable: + type: boolean + socket.max.fails: + type: integer + socket.nagle.disable: + type: boolean + socket.receive.buffer.bytes: + type: integer + socket.send.buffer.bytes: + type: integer + socket.timeout.ms: + type: integer + socket_cb: + type: string + ssl.ca.location: + type: string + ssl.ca.pem: + type: string + ssl.certificate.location: + type: string + ssl.certificate.pem: + type: string + ssl.cipher.suites: + type: string + ssl.crl.location: + type: string + ssl.curves.list: + type: string + ssl.endpoint.identification.algorithm: + type: string + ssl.engine.id: + type: string + ssl.engine.location: + type: string + ssl.key.location: + type: string + ssl.key.password: + type: string + ssl.key.pem: + type: string + ssl.keystore.location: + type: string + ssl.keystore.password: + type: string + ssl.providers: + type: string + ssl.sigalgs.list: + type: string + statistics.interval.ms: + type: integer + stats_cb: + type: string + throttle_cb: + type: string + topic.blacklist: + type: string + topic.metadata.propagation.max.ms: + type: integer + topic.metadata.refresh.fast.interval.ms: + type: integer + topic.metadata.refresh.interval.ms: + type: integer + topic.metadata.refresh.sparse: + type: boolean + type: object required_acks: type: integer sasl_over_ssl: diff --git a/docs/configuration/plugins/outputs/kafka.md b/docs/configuration/plugins/outputs/kafka.md index e0f29a642..0ff4629c2 100644 --- a/docs/configuration/plugins/outputs/kafka.md +++ b/docs/configuration/plugins/outputs/kafka.md @@ -184,6 +184,11 @@ Password when using PLAIN/SCRAM SASL authentication ### principal (string, optional) {#kafka-principal} +### rdkafka_options (*RdkafkaOptions, optional) {#kafka-rdkafka_options} + +RdkafkaOptions represents the global configuration properties for librdkafka. + + ### required_acks (int, optional) {#kafka-required_acks} The number of acks required per request . @@ -260,3 +265,468 @@ Username when using PLAIN/SCRAM SASL authentication +## RdkafkaOptions + +RdkafkaOptions represents the global configuration properties for librdkafka. + +### allow.auto.create.topics (bool, optional) {#rdkafkaoptions-allow.auto.create.topics} + +Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. The broker must also be configured with `auto.create.topics.enable=true` for this configuration to take effect. Note: the default value (true) for the producer is different from the default value (false) for the consumer. Further, the consumer default value is different from the Java consumer (true), and this property is not supported by the Java producer. Requires broker version >= 0.11.0.0, for older broker versions only the broker configuration applies. + + +### api.version.fallback.ms (int, optional) {#rdkafkaoptions-api.version.fallback.ms} + +Dictates how long the `broker.version.fallback` fallback is used in the case the ApiVersionRequest fails. + + +### api.version.request (bool, optional) {#rdkafkaoptions-api.version.request} + +Request broker's supported API versions to adjust functionality to available protocol features. If set to false, or the ApiVersionRequest fails, the fallback version `broker.version.fallback` will be used. **NOTE**: Depends on broker version >=0.10.0. If the request is not supported by (an older) broker the `broker.version.fallback` fallback is used. + + +### api.version.request.timeout.ms (int, optional) {#rdkafkaoptions-api.version.request.timeout.ms} + +Timeout for broker API version requests. + + +### background_event_cb (string, optional) {#rdkafkaoptions-background_event_cb} + +Background queue event callback (set with rd_kafka_conf_set_background_event_cb()) + + +### bootstrap.servers (string, optional) {#rdkafkaoptions-bootstrap.servers} + +Alias for `metadata.broker.list`: Initial list of brokers as a CSV list of broker host or host:port. The application may also use `rd_kafka_brokers_add()` to add brokers during runtime. + + +### broker.address.family (string, optional) {#rdkafkaoptions-broker.address.family} + +Allowed broker IP address families: any, v4, v6 + + +### broker.address.ttl (int, optional) {#rdkafkaoptions-broker.address.ttl} + +How long to cache the broker address resolving results (milliseconds). + + +### broker.version.fallback (string, optional) {#rdkafkaoptions-broker.version.fallback} + +Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see `api.version.request`) making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for `api.version.fallback.ms`. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value >= 0.10, such as 0.10.2.1, enables ApiVersionRequests. + + +### builtin.features (string, optional) {#rdkafkaoptions-builtin.features} + +Indicates the builtin features for this build of librdkafka. An application can either query this value or attempt to set it with its list of required features to check for library support. + + +### client.id (string, optional) {#rdkafkaoptions-client.id} + +Client identifier. + + +### closesocket_cb (string, optional) {#rdkafkaoptions-closesocket_cb} + +Socket close callback + + +### connect_cb (string, optional) {#rdkafkaoptions-connect_cb} + +Socket connect callback + + +### connections.max.idle.ms (int, optional) {#rdkafkaoptions-connections.max.idle.ms} + +Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info). + + +### debug (string, optional) {#rdkafkaoptions-debug} + +A comma-separated list of debug contexts to enable. Detailed Producer debugging: broker,topic,msg. Consumer: consumer,cgrp,topic,fetch + + +### default_topic_conf (string, optional) {#rdkafkaoptions-default_topic_conf} + +Default topic configuration for automatically subscribed topics + + +### enable.random.seed (bool, optional) {#rdkafkaoptions-enable.random.seed} + +If enabled librdkafka will initialize the PRNG with srand(current_time.milliseconds) on the first invocation of rd_kafka_new() (required only if rand_r() is not available on your platform). If disabled the application must call srand() prior to calling rd_kafka_new(). + + +### enable.ssl.certificate.verification (bool, optional) {#rdkafkaoptions-enable.ssl.certificate.verification} + +Enable OpenSSL's builtin broker (server) certificate verification. This verification can be extended by the application by implementing a certificate_verify_cb. + + +### enable.sasl.oauthbearer.unsecure.jwt (bool, optional) {#rdkafkaoptions-enable.sasl.oauthbearer.unsecure.jwt} + +Enable the builtin unsecure JWT OAUTHBEARER token handler if no oauthbearer_refresh_cb has been set. This builtin handler should only be used for development or testing, and not in production. + + +### enabled_events (int, optional) {#rdkafkaoptions-enabled_events} + +See `rd_kafka_conf_set_events()` + + +### error_cb (string, optional) {#rdkafkaoptions-error_cb} + +Error callback (set with rd_kafka_conf_set_error_cb()) + + +### interceptors (string, optional) {#rdkafkaoptions-interceptors} + +Interceptors added through rd_kafka_conf_interceptor_add_..() and any configuration handled by interceptors. + + +### internal.termination.signal (int, optional) {#rdkafkaoptions-internal.termination.signal} + +Signal that librdkafka will use to quickly terminate on rd_kafka_destroy(). If this signal is not set then there will be a delay before rd_kafka_wait_destroyed() returns true as internal threads are timing out their system calls. If this signal is set however the delay will be minimal. The application should mask this signal as an internal signal handler is installed. + + +### log_cb (string, optional) {#rdkafkaoptions-log_cb} + +Log callback (set with rd_kafka_conf_set_log_cb()) + + +### log.connection.close (bool, optional) {#rdkafkaoptions-log.connection.close} + +Log broker disconnects. It might be useful to turn this off when interacting with 0.9 brokers with an aggressive `connections.max.idle.ms` value. + + +### log_level (int, optional) {#rdkafkaoptions-log_level} + +Logging level (syslog(3) levels) + + +### log.queue (bool, optional) {#rdkafkaoptions-log.queue} + +Disable spontaneous log_cb from internal librdkafka threads, instead enqueue log messages on queue set with `rd_kafka_set_log_queue()` and serve log callbacks or events through the standard poll APIs. **NOTE**: Log messages will linger in a temporary queue until the log queue has been set. + + +### log.thread.name (bool, optional) {#rdkafkaoptions-log.thread.name} + +Print internal thread name in log messages (useful for debugging librdkafka internals) + + +### max.in.flight (int, optional) {#rdkafkaoptions-max.in.flight} + +Alias for `max.in.flight.requests.per.connection`: Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. + + +### max.in.flight.requests.per.connection (int, optional) {#rdkafkaoptions-max.in.flight.requests.per.connection} + +Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. + + +### message.copy.max.bytes (int, optional) {#rdkafkaoptions-message.copy.max.bytes} + +Maximum size for message to be copied to buffer. Messages larger than this will be passed by reference (zero-copy) at the expense of larger iovecs. + + +### message.max.bytes (int, optional) {#rdkafkaoptions-message.max.bytes} + +Maximum Kafka protocol request message size. Due to differing framing overhead between protocol versions the producer is unable to reliably enforce a strict max message limit at produce time and may exceed the maximum size by one message in protocol ProduceRequests, the broker will enforce the the topic's `max.message.bytes` limit (see Apache Kafka documentation). + + +### metadata.broker.list (string, optional) {#rdkafkaoptions-metadata.broker.list} + +Initial list of brokers as a CSV list of broker host or host:port. The application may also use `rd_kafka_brokers_add()` to add brokers during runtime. + + +### metadata.max.age.ms (int, optional) {#rdkafkaoptions-metadata.max.age.ms} + +Metadata cache max age. Defaults to topic.metadata.refresh.interval.ms * 3 + + +### oauthbearer_token_refresh_cb (string, optional) {#rdkafkaoptions-oauthbearer_token_refresh_cb} + +SASL/OAUTHBEARER token refresh callback (set with rd_kafka_conf_set_oauthbearer_token_refresh_cb(), triggered by rd_kafka_poll(), et.al. This callback will be triggered when it is time to refresh the client's OAUTHBEARER token. Also see rd_kafka_conf_enable_sasl_queue(). + + +### opaque (string, optional) {#rdkafkaoptions-opaque} + +Application opaque (set with rd_kafka_conf_set_opaque()) + + +### open_cb (string, optional) {#rdkafkaoptions-open_cb} + +File open callback to provide race-free CLOEXEC + + +### plugin.library.paths (string, optional) {#rdkafkaoptions-plugin.library.paths} + +List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically. + + +### receive.message.max.bytes (int, optional) {#rdkafkaoptions-receive.message.max.bytes} + +Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hickups. This value must be at least `fetch.max.bytes` + 512 to allow for protocol overhead; the value is adjusted automatically unless the configuration property is explicitly set. + + +### reconnect.backoff.max.ms (int, optional) {#rdkafkaoptions-reconnect.backoff.max.ms} + +The maximum time to wait before reconnecting to a broker after the connection has been closed. + + +### reconnect.backoff.ms (int, optional) {#rdkafkaoptions-reconnect.backoff.ms} + +The initial time to wait before reconnecting to a broker after the connection has been closed. The time is increased exponentially until `reconnect.backoff.max.ms` is reached. -25% to +50% jitter is applied to each reconnect backoff. A value of 0 disables the backoff and reconnects immediately. + + +### resolve_cb (string, optional) {#rdkafkaoptions-resolve_cb} + +Address resolution callback (set with rd_kafka_conf_set_resolve_cb()) + + +### ssl.ca.location (string, optional) {#rdkafkaoptions-ssl.ca.location} + +File or directory path to CA certificate(s) for verifying the broker's key. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to `probe`. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or `ssl.ca.location` is set to `probe` a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see `OPENSSLDIR` in `openssl version -a`). + + +### ssl.ca.pem (string, optional) {#rdkafkaoptions-ssl.ca.pem} + +CA certificate string (PEM format) for verifying the broker's key. + + +### ssl.certificate.location (string, optional) {#rdkafkaoptions-ssl.certificate.location} + +Path to client's public key (PEM) used for authentication. + + +### ssl.certificate.pem (string, optional) {#rdkafkaoptions-ssl.certificate.pem} + +Client's public key string (PEM format) used for authentication. + + +### ssl.cipher.suites (string, optional) {#rdkafkaoptions-ssl.cipher.suites} + +A cipher suite is a named combination of authentication, encryption, MAC and key exchange algorithm used to negotiate the security settings for a network connection using TLS or SSL network protocol. See manual page for `ciphers(1)` and `SSL_CTX_set_cipher_list(3). + + +### ssl.crl.location (string, optional) {#rdkafkaoptions-ssl.crl.location} + +Path to CRL for verifying broker's certificate validity. + + +### ssl.curves.list (string, optional) {#rdkafkaoptions-ssl.curves.list} + +The supported-curves extension in the TLS ClientHello message specifies the curves (standard/named, or 'explicit' GF(2^k) or GF(p)) the client is willing to have the server use. See manual page for `SSL_CTX_set1_curves_list(3)`. OpenSSL >= 1.0.2 required. + + +### ssl.endpoint.identification.algorithm (string, optional) {#rdkafkaoptions-ssl.endpoint.identification.algorithm} + +Endpoint identification algorithm to validate broker hostname using broker certificate. https - Server (broker) hostname verification as specified in RFC2818. none - No endpoint verification. OpenSSL >= 1.0.2 required. + + +### ssl.engine.id (string, optional) {#rdkafkaoptions-ssl.engine.id} + +OpenSSL engine id is the name used for loading engine. + + +### ssl.engine.location (string, optional) {#rdkafkaoptions-ssl.engine.location} + +**DEPRECATED** Path to OpenSSL engine library. OpenSSL >= 1.1.x required. DEPRECATED: OpenSSL engine support is deprecated and should be replaced by OpenSSL 3 providers. + + +### ssl.key.location (string, optional) {#rdkafkaoptions-ssl.key.location} + +Path to client's private key (PEM) used for authentication. + + +### ssl.key.password (string, optional) {#rdkafkaoptions-ssl.key.password} + +Private key passphrase (for use with `ssl.key.location` and `set_ssl_cert()`). + + +### ssl.key.pem (string, optional) {#rdkafkaoptions-ssl.key.pem} + +Client's private key string (PEM format) used for authentication. + + +### ssl.keystore.location (string, optional) {#rdkafkaoptions-ssl.keystore.location} + +Path to client's keystore (PKCS#12) used for authentication. + + +### ssl.keystore.password (string, optional) {#rdkafkaoptions-ssl.keystore.password} + +Client's keystore (PKCS#12) password. + + +### ssl.providers (string, optional) {#rdkafkaoptions-ssl.providers} + +Comma-separated list of OpenSSL 3.0.x implementation providers. E.g., "default,legacy". + + +### ssl.sigalgs.list (string, optional) {#rdkafkaoptions-ssl.sigalgs.list} + +The client uses the TLS ClientHello signature_algorithms extension to indicate to the server which signature/hash algorithm pairs may be used in digital signatures. See manual page for `SSL_CTX_set1_sigalgs_list(3)`. OpenSSL >= 1.0.2 required. + + +### sasl.kerberos.keytab (string, optional) {#rdkafkaoptions-sasl.kerberos.keytab} + +Path to Kerberos keytab file. This configuration property is only used as a variable in sasl.kerberos.kinit.cmd as ... -t "%{sasl.kerberos.keytab}". + + +### sasl.kerberos.kinit.cmd (string, optional) {#rdkafkaoptions-sasl.kerberos.kinit.cmd} + +Shell command to refresh or acquire the client's Kerberos ticket. This command is executed on client creation and every sasl.kerberos.min.time.before.relogin (0=disable). + + +### sasl.kerberos.min.time.before.relogin (int, optional) {#rdkafkaoptions-sasl.kerberos.min.time.before.relogin} + +Minimum time in milliseconds between key refresh attempts. Disable automatic key refresh by setting this property to 0. + + +### sasl.kerberos.principal (string, optional) {#rdkafkaoptions-sasl.kerberos.principal} + +This client's Kerberos principal name. (Not supported on Windows, will use the logon user's principal). + + +### sasl.kerberos.service.name (string, optional) {#rdkafkaoptions-sasl.kerberos.service.name} + +Kerberos principal name that Kafka runs as, not including /hostname@REALM. + + +### sasl.mechanisms (string, optional) {#rdkafkaoptions-sasl.mechanisms} + +SASL mechanism to use for authentication. Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER. NOTE: Despite the name only one mechanism must be configured. + + +### sasl.oauthbearer.client.id (string, optional) {#rdkafkaoptions-sasl.oauthbearer.client.id} + +Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when sasl.oauthbearer.method is set to "oidc". + + +### sasl.oauthbearer.client.secret (string, optional) {#rdkafkaoptions-sasl.oauthbearer.client.secret} + +Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when sasl.oauthbearer.method is set to "oidc". + + +### sasl.oauthbearer.config (string, optional) {#rdkafkaoptions-sasl.oauthbearer.config} + +SASL/OAUTHBEARER configuration. The format is implementation-dependent and must be parsed accordingly. The default unsecured token implementation (see https://tools.ietf.org/html/rfc7515#appendix-A.5) recognizes space-separated name=value pairs with valid names including principalClaimName, principal, scopeClaimName, scope, and lifeSeconds. The default value for principalClaimName is "sub", the default value for scopeClaimName is "scope", and the default value for lifeSeconds is 3600. The scope value is CSV format with the default value being no/empty scope. For example: principalClaimName=azp principal=admin scopeClaimName=roles scope=role1,role2 lifeSeconds=600. In addition, SASL extensions can be communicated to the broker via extension_NAME=value. For example: principal=admin extension_traceId=123. + + +### sasl.oauthbearer.extensions (string, optional) {#rdkafkaoptions-sasl.oauthbearer.extensions} + +Allow additional information to be provided to the broker. Comma-separated list of key=value pairs. E.g., "supportFeatureX=true,organizationId=sales-emea".Only used when sasl.oauthbearer.method is set to "oidc". + + +### sasl.oauthbearer.method (string, optional) {#rdkafkaoptions-sasl.oauthbearer.method} + +Set to "default" or "oidc" to control which login method to be used. If set to "oidc", the following properties must also be specified: sasl.oauthbearer.client.id, sasl.oauthbearer.client.secret, and sasl.oauthbearer.token.endpoint.url. + + +### sasl.oauthbearer.scope (string, optional) {#rdkafkaoptions-sasl.oauthbearer.scope} + +Client use this to specify the scope of the access request to the broker. Only used when sasl.oauthbearer.method is set to "oidc". + + +### sasl.oauthbearer.token.endpoint.url (string, optional) {#rdkafkaoptions-sasl.oauthbearer.token.endpoint.url} + +OAuth/OIDC issuer token endpoint HTTP(S) URI used to retrieve token. Only used when sasl.oauthbearer.method is set to "oidc". + + +### sasl.password (string, optional) {#rdkafkaoptions-sasl.password} + +SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism. + + +### sasl.username (string, optional) {#rdkafkaoptions-sasl.username} + +SASL username for use with the PLAIN and SASL-SCRAM-.. mechanisms. + + +### security.protocol (string, optional) {#rdkafkaoptions-security.protocol} + +Protocol used to communicate with brokers. + + +### socket.blocking.max.ms (int, optional) {#rdkafkaoptions-socket.blocking.max.ms} + +DEPRECATED No longer used. + + +### socket_cb (string, optional) {#rdkafkaoptions-socket_cb} + +Socket creation callback to provide race-free CLOEXEC + + +### socket.connection.setup.timeout.ms (int, optional) {#rdkafkaoptions-socket.connection.setup.timeout.ms} + +Maximum time allowed for broker connection setup (TCP connection setup as well SSL and SASL handshake). If the connection to the broker is not fully functional after this the connection will be closed and retried. + + +### socket.keepalive.enable (bool, optional) {#rdkafkaoptions-socket.keepalive.enable} + +Enable TCP keep-alives (SO_KEEPALIVE) on broker sockets + + +### socket.max.fails (int, optional) {#rdkafkaoptions-socket.max.fails} + +Disconnect from broker when this number of send failures (e.g., timed out requests) is reached. Disable with 0. WARNING: It is highly recommended to leave this setting at its default value of 1 to avoid the client and broker to become desynchronized in case of request timeouts. NOTE: The connection is automatically re-established. + + +### socket.nagle.disable (bool, optional) {#rdkafkaoptions-socket.nagle.disable} + +Disable the Nagle algorithm (TCP_NODELAY) on broker sockets. + + +### socket.receive.buffer.bytes (int, optional) {#rdkafkaoptions-socket.receive.buffer.bytes} + +Broker socket receive buffer size. System default is used if 0. + + +### socket.send.buffer.bytes (int, optional) {#rdkafkaoptions-socket.send.buffer.bytes} + +Broker socket send buffer size. System default is used if 0. + + +### socket.timeout.ms (int, optional) {#rdkafkaoptions-socket.timeout.ms} + +Default timeout for network requests. Producer: ProduceRequests will use the lesser value of `socket.timeout.ms` and remaining `message.timeout.ms` for the first message in the batch. Consumer: FetchRequests will use `fetch.wait.max.ms` + `socket.timeout.ms`. Admin: Admin requests will use `socket.timeout.ms` or explicitly set `rd_kafka_AdminOptions_set_operation_timeout()` value. + + +### statistics.interval.ms (int, optional) {#rdkafkaoptions-statistics.interval.ms} + +librdkafka statistics emit interval. The application also needs to register a stats callback using `rd_kafka_conf_set_stats_cb()`. The granularity is 1000ms. A value of 0 disables statistics. + + +### stats_cb (string, optional) {#rdkafkaoptions-stats_cb} + +Statistics callback (set with rd_kafka_conf_set_stats_cb()) + + +### throttle_cb (string, optional) {#rdkafkaoptions-throttle_cb} + +Throttle callback (set with rd_kafka_conf_set_throttle_cb()) + + +### topic.blacklist (string, optional) {#rdkafkaoptions-topic.blacklist} + +Topic blacklist, a comma-separated list of regular expressions for matching topic names that should be ignored in broker metadata information as if the topics did not exist. + + +### topic.metadata.propagation.max.ms (int, optional) {#rdkafkaoptions-topic.metadata.propagation.max.ms} + +Apache Kafka topic creation is asynchronous and it takes some time for a new topic to propagate throughout the cluster to all brokers. If a client requests topic metadata after manual topic creation but before the topic has been fully propagated to the broker the client is requesting metadata from, the topic will seem to be non-existent and the client will mark the topic as such, failing queued produced messages with `ERR__UNKNOWN_TOPIC`. This setting delays marking a topic as non-existent until the configured propagation max time has passed. The maximum propagation time is calculated from the time the topic is first referenced in the client, e.g., on produce(). + + +### topic.metadata.refresh.fast.interval.ms (int, optional) {#rdkafkaoptions-topic.metadata.refresh.fast.interval.ms} + +When a topic loses its leader a new metadata request will be enqueued immediately and then with this initial interval, exponentially increasing upto `retry.backoff.max.ms`, until the topic metadata has been refreshed. If not set explicitly, it will be defaulted to `retry.backoff.ms`. This is used to recover quickly from transitioning leader brokers. + + +### topic.metadata.refresh.interval.ms (int, optional) {#rdkafkaoptions-topic.metadata.refresh.interval.ms} + +Period of time in milliseconds at which topic and broker metadata is refreshed in order to proactively discover any new brokers, topics, partitions or partition leader changes. Use -1 to disable the intervalled refresh (not recommended). If there are no locally referenced topics (no topic objects created, no messages produced, no subscription or no assignment) then only the broker list will be refreshed every interval but no more often than every 10s. + + +### topic.metadata.refresh.sparse (bool, optional) {#rdkafkaoptions-topic.metadata.refresh.sparse} + +Sparse metadata requests (consumes less network bandwidth) + + + diff --git a/pkg/sdk/logging/model/output/kafka.go b/pkg/sdk/logging/model/output/kafka.go index fcbb06aef..afa32a659 100644 --- a/pkg/sdk/logging/model/output/kafka.go +++ b/pkg/sdk/logging/model/output/kafka.go @@ -66,6 +66,8 @@ type _metaKafka interface{} //nolint:deadcode,unused type KafkaOutputConfig struct { // Use rdkafka2 instead of the legacy kafka2 output plugin. This plugin requires fluentd image version v1.16-4.9-full or higher. UseRdkafka bool `json:"use_rdkafka,omitempty"` + // RdkafkaOptions represents the global configuration properties for librdkafka. + RdkafkaOptions *RdkafkaOptions `json:"rdkafka_options,omitempty"` // The list of all seed brokers, with their host and port information. Brokers string `json:"brokers"` // Topic Key (default: "topic") @@ -147,6 +149,194 @@ type KafkaOutputConfig struct { SlowFlushLogThreshold string `json:"slow_flush_log_threshold,omitempty"` } +// RdkafkaOptions represents the global configuration properties for librdkafka. +type RdkafkaOptions struct { + // Indicates the builtin features for this build of librdkafka. An application can either query this value or attempt to set it with its list of required features to check for library support. + BuiltinFeatures string `json:"builtin.features,omitempty"` + // Client identifier. + ClientID string `json:"client.id,omitempty"` + // Initial list of brokers as a CSV list of broker host or host:port. The application may also use `rd_kafka_brokers_add()` to add brokers during runtime. + MetadataBrokerList string `json:"metadata.broker.list,omitempty"` + // Alias for `metadata.broker.list`: Initial list of brokers as a CSV list of broker host or host:port. The application may also use `rd_kafka_brokers_add()` to add brokers during runtime. + BootstrapServers string `json:"bootstrap.servers,omitempty"` + // Maximum Kafka protocol request message size. Due to differing framing overhead between protocol versions the producer is unable to reliably enforce a strict max message limit at produce time and may exceed the maximum size by one message in protocol ProduceRequests, the broker will enforce the the topic's `max.message.bytes` limit (see Apache Kafka documentation). + MessageMaxBytes int `json:"message.max.bytes,omitempty"` + // Maximum size for message to be copied to buffer. Messages larger than this will be passed by reference (zero-copy) at the expense of larger iovecs. + MessageCopyMaxBytes int `json:"message.copy.max.bytes,omitempty"` + // Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hickups. This value must be at least `fetch.max.bytes` + 512 to allow for protocol overhead; the value is adjusted automatically unless the configuration property is explicitly set. + ReceiveMessageMaxBytes int `json:"receive.message.max.bytes,omitempty"` + // Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. + MaxInFlightRequestsPerConnection int `json:"max.in.flight.requests.per.connection,omitempty"` + // Alias for `max.in.flight.requests.per.connection`: Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. + MaxInFlight int `json:"max.in.flight,omitempty"` + // Period of time in milliseconds at which topic and broker metadata is refreshed in order to proactively discover any new brokers, topics, partitions or partition leader changes. Use -1 to disable the intervalled refresh (not recommended). If there are no locally referenced topics (no topic objects created, no messages produced, no subscription or no assignment) then only the broker list will be refreshed every interval but no more often than every 10s. + TopicMetadataRefreshIntervalMs int `json:"topic.metadata.refresh.interval.ms,omitempty"` + // Metadata cache max age. Defaults to topic.metadata.refresh.interval.ms * 3 + MetadataMaxAgeMs int `json:"metadata.max.age.ms,omitempty"` + // When a topic loses its leader a new metadata request will be enqueued immediately and then with this initial interval, exponentially increasing upto `retry.backoff.max.ms`, until the topic metadata has been refreshed. If not set explicitly, it will be defaulted to `retry.backoff.ms`. This is used to recover quickly from transitioning leader brokers. + TopicMetadataRefreshFastIntervalMs int `json:"topic.metadata.refresh.fast.interval.ms,omitempty"` + // Sparse metadata requests (consumes less network bandwidth) + TopicMetadataRefreshSparse bool `json:"topic.metadata.refresh.sparse,omitempty"` + // Apache Kafka topic creation is asynchronous and it takes some time for a new topic to propagate throughout the cluster to all brokers. If a client requests topic metadata after manual topic creation but before the topic has been fully propagated to the broker the client is requesting metadata from, the topic will seem to be non-existent and the client will mark the topic as such, failing queued produced messages with `ERR__UNKNOWN_TOPIC`. This setting delays marking a topic as non-existent until the configured propagation max time has passed. The maximum propagation time is calculated from the time the topic is first referenced in the client, e.g., on produce(). + TopicMetadataPropagationMaxMs int `json:"topic.metadata.propagation.max.ms,omitempty"` + // Topic blacklist, a comma-separated list of regular expressions for matching topic names that should be ignored in broker metadata information as if the topics did not exist. + TopicBlacklist string `json:"topic.blacklist,omitempty"` + // A comma-separated list of debug contexts to enable. Detailed Producer debugging: broker,topic,msg. Consumer: consumer,cgrp,topic,fetch + Debug string `json:"debug,omitempty"` + // Default timeout for network requests. Producer: ProduceRequests will use the lesser value of `socket.timeout.ms` and remaining `message.timeout.ms` for the first message in the batch. Consumer: FetchRequests will use `fetch.wait.max.ms` + `socket.timeout.ms`. Admin: Admin requests will use `socket.timeout.ms` or explicitly set `rd_kafka_AdminOptions_set_operation_timeout()` value. + SocketTimeoutMs int `json:"socket.timeout.ms,omitempty"` + // DEPRECATED No longer used. + SocketBlockingMaxMs int `json:"socket.blocking.max.ms,omitempty"` + // Broker socket send buffer size. System default is used if 0. + SocketSendBufferBytes int `json:"socket.send.buffer.bytes,omitempty"` + // Broker socket receive buffer size. System default is used if 0. + SocketReceiveBufferBytes int `json:"socket.receive.buffer.bytes,omitempty"` + // Enable TCP keep-alives (SO_KEEPALIVE) on broker sockets + SocketKeepaliveEnable bool `json:"socket.keepalive.enable,omitempty"` + // Disable the Nagle algorithm (TCP_NODELAY) on broker sockets. + SocketNagleDisable bool `json:"socket.nagle.disable,omitempty"` + // Disconnect from broker when this number of send failures (e.g., timed out requests) is reached. Disable with 0. WARNING: It is highly recommended to leave this setting at its default value of 1 to avoid the client and broker to become desynchronized in case of request timeouts. NOTE: The connection is automatically re-established. + SocketMaxFails int `json:"socket.max.fails,omitempty"` + // How long to cache the broker address resolving results (milliseconds). + BrokerAddressTTl int `json:"broker.address.ttl,omitempty"` + // Allowed broker IP address families: any, v4, v6 + BrokerAddressFamily string `json:"broker.address.family,omitempty"` + // Maximum time allowed for broker connection setup (TCP connection setup as well SSL and SASL handshake). If the connection to the broker is not fully functional after this the connection will be closed and retried. + SocketConnectionSetupTimeoutMs int `json:"socket.connection.setup.timeout.ms,omitempty"` + // Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info). + ConnectionsMaxIdleMs int `json:"connections.max.idle.ms,omitempty"` + // The initial time to wait before reconnecting to a broker after the connection has been closed. The time is increased exponentially until `reconnect.backoff.max.ms` is reached. -25% to +50% jitter is applied to each reconnect backoff. A value of 0 disables the backoff and reconnects immediately. + ReconnectBackoffMs int `json:"reconnect.backoff.ms,omitempty"` + // The maximum time to wait before reconnecting to a broker after the connection has been closed. + ReconnectBackoffMaxMs int `json:"reconnect.backoff.max.ms,omitempty"` + // librdkafka statistics emit interval. The application also needs to register a stats callback using `rd_kafka_conf_set_stats_cb()`. The granularity is 1000ms. A value of 0 disables statistics. + StatisticsIntervalMs int `json:"statistics.interval.ms,omitempty"` + // See `rd_kafka_conf_set_events()` + EnabledEvents int `json:"enabled_events,omitempty"` + // Error callback (set with rd_kafka_conf_set_error_cb()) + ErrorCb string `json:"error_cb,omitempty"` + // Throttle callback (set with rd_kafka_conf_set_throttle_cb()) + ThrottleCb string `json:"throttle_cb,omitempty"` + // Statistics callback (set with rd_kafka_conf_set_stats_cb()) + StatsCb string `json:"stats_cb,omitempty"` + // Log callback (set with rd_kafka_conf_set_log_cb()) + LogCb string `json:"log_cb,omitempty"` + // Logging level (syslog(3) levels) + LogLevel int `json:"log_level,omitempty"` + // Disable spontaneous log_cb from internal librdkafka threads, instead enqueue log messages on queue set with `rd_kafka_set_log_queue()` and serve log callbacks or events through the standard poll APIs. **NOTE**: Log messages will linger in a temporary queue until the log queue has been set. + LogQueue bool `json:"log.queue,omitempty"` + // Print internal thread name in log messages (useful for debugging librdkafka internals) + LogThreadName bool `json:"log.thread.name,omitempty"` + // If enabled librdkafka will initialize the PRNG with srand(current_time.milliseconds) on the first invocation of rd_kafka_new() (required only if rand_r() is not available on your platform). If disabled the application must call srand() prior to calling rd_kafka_new(). + EnableRandomSeed bool `json:"enable.random.seed,omitempty"` + // Log broker disconnects. It might be useful to turn this off when interacting with 0.9 brokers with an aggressive `connections.max.idle.ms` value. + LogConnectionClose bool `json:"log.connection.close,omitempty"` + // Background queue event callback (set with rd_kafka_conf_set_background_event_cb()) + BackgroundEventCb string `json:"background_event_cb,omitempty"` + // Socket creation callback to provide race-free CLOEXEC + SocketCb string `json:"socket_cb,omitempty"` + // Socket connect callback + ConnectCb string `json:"connect_cb,omitempty"` + // Socket close callback + CloseSocketCb string `json:"closesocket_cb,omitempty"` + // File open callback to provide race-free CLOEXEC + OpenCb string `json:"open_cb,omitempty"` + // Address resolution callback (set with rd_kafka_conf_set_resolve_cb()) + ResolveCb string `json:"resolve_cb,omitempty"` + // Application opaque (set with rd_kafka_conf_set_opaque()) + Opaque string `json:"opaque,omitempty"` + // Default topic configuration for automatically subscribed topics + DefaultTopicConf string `json:"default_topic_conf,omitempty"` + // Signal that librdkafka will use to quickly terminate on rd_kafka_destroy(). If this signal is not set then there will be a delay before rd_kafka_wait_destroyed() returns true as internal threads are timing out their system calls. If this signal is set however the delay will be minimal. The application should mask this signal as an internal signal handler is installed. + InternalTerminationSignal int `json:"internal.termination.signal,omitempty"` + // Request broker's supported API versions to adjust functionality to available protocol features. If set to false, or the ApiVersionRequest fails, the fallback version `broker.version.fallback` will be used. **NOTE**: Depends on broker version >=0.10.0. If the request is not supported by (an older) broker the `broker.version.fallback` fallback is used. + ApiVersionRequest bool `json:"api.version.request,omitempty"` + // Timeout for broker API version requests. + ApiVersionRequestTimeoutMs int `json:"api.version.request.timeout.ms,omitempty"` + // Dictates how long the `broker.version.fallback` fallback is used in the case the ApiVersionRequest fails. + ApiVersionFallbackMs int `json:"api.version.fallback.ms,omitempty"` + // Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see `api.version.request`) making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for `api.version.fallback.ms`. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value >= 0.10, such as 0.10.2.1, enables ApiVersionRequests. + BrokerVersionFallback string `json:"broker.version.fallback,omitempty"` + // Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. The broker must also be configured with `auto.create.topics.enable=true` for this configuration to take effect. Note: the default value (true) for the producer is different from the default value (false) for the consumer. Further, the consumer default value is different from the Java consumer (true), and this property is not supported by the Java producer. Requires broker version >= 0.11.0.0, for older broker versions only the broker configuration applies. + AllowAutoCreateTopics bool `json:"allow.auto.create.topics,omitempty"` + // Protocol used to communicate with brokers. + SecurityProtocol string `json:"security.protocol,omitempty"` + // A cipher suite is a named combination of authentication, encryption, MAC and key exchange algorithm used to negotiate the security settings for a network connection using TLS or SSL network protocol. See manual page for `ciphers(1)` and `SSL_CTX_set_cipher_list(3). + SSLCipherSuites string `json:"ssl.cipher.suites,omitempty"` + // The supported-curves extension in the TLS ClientHello message specifies the curves (standard/named, or 'explicit' GF(2^k) or GF(p)) the client is willing to have the server use. See manual page for `SSL_CTX_set1_curves_list(3)`. OpenSSL >= 1.0.2 required. + SSLCurvesList string `json:"ssl.curves.list,omitempty"` + // The client uses the TLS ClientHello signature_algorithms extension to indicate to the server which signature/hash algorithm pairs may be used in digital signatures. See manual page for `SSL_CTX_set1_sigalgs_list(3)`. OpenSSL >= 1.0.2 required. + SSLSigalgsList string `json:"ssl.sigalgs.list,omitempty"` + // Path to client's private key (PEM) used for authentication. + SSLKeyLocation string `json:"ssl.key.location,omitempty"` + // Private key passphrase (for use with `ssl.key.location` and `set_ssl_cert()`). + SSLKeyPassword string `json:"ssl.key.password,omitempty"` + // Client's private key string (PEM format) used for authentication. + SSLKeyPem string `json:"ssl.key.pem,omitempty"` + // Path to client's public key (PEM) used for authentication. + SSLCertificateLocation string `json:"ssl.certificate.location,omitempty"` + // Client's public key string (PEM format) used for authentication. + SSLCertificatePem string `json:"ssl.certificate.pem,omitempty"` + // File or directory path to CA certificate(s) for verifying the broker's key. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to `probe`. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or `ssl.ca.location` is set to `probe` a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see `OPENSSLDIR` in `openssl version -a`). + SSLCaLocation string `json:"ssl.ca.location,omitempty"` + // CA certificate string (PEM format) for verifying the broker's key. + SSLCaPem string `json:"ssl.ca.pem,omitempty"` + // Path to CRL for verifying broker's certificate validity. + SSLCrlLocation string `json:"ssl.crl.location,omitempty"` + // Path to client's keystore (PKCS#12) used for authentication. + SSLKeystoreLocation string `json:"ssl.keystore.location,omitempty"` + // Client's keystore (PKCS#12) password. + SSLKeystorePassword string `json:"ssl.keystore.password,omitempty"` + // Comma-separated list of OpenSSL 3.0.x implementation providers. E.g., "default,legacy". + SSLProviders string `json:"ssl.providers,omitempty"` + // **DEPRECATED** Path to OpenSSL engine library. OpenSSL >= 1.1.x required. DEPRECATED: OpenSSL engine support is deprecated and should be replaced by OpenSSL 3 providers. + SSLEngineLocation string `json:"ssl.engine.location,omitempty"` + // OpenSSL engine id is the name used for loading engine. + SSLEngineId string `json:"ssl.engine.id,omitempty"` + // Enable OpenSSL's builtin broker (server) certificate verification. This verification can be extended by the application by implementing a certificate_verify_cb. + EnableSSLCertificateVerification bool `json:"enable.ssl.certificate.verification,omitempty"` + // Endpoint identification algorithm to validate broker hostname using broker certificate. https - Server (broker) hostname verification as specified in RFC2818. none - No endpoint verification. OpenSSL >= 1.0.2 required. + SSLEndpointIdentificationAlgorithm string `json:"ssl.endpoint.identification.algorithm,omitempty"` + // SASL mechanism to use for authentication. Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER. NOTE: Despite the name only one mechanism must be configured. + SaslMechanisms string `json:"sasl.mechanisms,omitempty" ` + // Kerberos principal name that Kafka runs as, not including /hostname@REALM. + SaslKerberosServiceName string `json:"sasl.kerberos.service.name,omitempty" ` + // This client's Kerberos principal name. (Not supported on Windows, will use the logon user's principal). + SaslKerberosPrincipal string `json:"sasl.kerberos.principal,omitempty" ` + // Shell command to refresh or acquire the client's Kerberos ticket. This command is executed on client creation and every sasl.kerberos.min.time.before.relogin (0=disable). + SaslKerberosKinitCmd string `json:"sasl.kerberos.kinit.cmd,omitempty" ` + // Path to Kerberos keytab file. This configuration property is only used as a variable in sasl.kerberos.kinit.cmd as ... -t "%{sasl.kerberos.keytab}". + SaslKerberosKeytab string `json:"sasl.kerberos.keytab,omitempty" ` + // Minimum time in milliseconds between key refresh attempts. Disable automatic key refresh by setting this property to 0. + SaslKerberosMinTimeBeforeRelogin int `json:"sasl.kerberos.min.time.before.relogin,omitempty" ` + // SASL username for use with the PLAIN and SASL-SCRAM-.. mechanisms. + SaslUsername string `json:"sasl.username,omitempty" ` + // SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism. + SaslPassword string `json:"sasl.password,omitempty" ` + // SASL/OAUTHBEARER configuration. The format is implementation-dependent and must be parsed accordingly. The default unsecured token implementation (see https://tools.ietf.org/html/rfc7515#appendix-A.5) recognizes space-separated name=value pairs with valid names including principalClaimName, principal, scopeClaimName, scope, and lifeSeconds. The default value for principalClaimName is "sub", the default value for scopeClaimName is "scope", and the default value for lifeSeconds is 3600. The scope value is CSV format with the default value being no/empty scope. For example: principalClaimName=azp principal=admin scopeClaimName=roles scope=role1,role2 lifeSeconds=600. In addition, SASL extensions can be communicated to the broker via extension_NAME=value. For example: principal=admin extension_traceId=123. + SaslOauthbearerConfig string `json:"sasl.oauthbearer.config,omitempty" ` + // Enable the builtin unsecure JWT OAUTHBEARER token handler if no oauthbearer_refresh_cb has been set. This builtin handler should only be used for development or testing, and not in production. + EnableSaslOauthbearerUnsecureJwt bool `json:"enable.sasl.oauthbearer.unsecure.jwt,omitempty" ` + // SASL/OAUTHBEARER token refresh callback (set with rd_kafka_conf_set_oauthbearer_token_refresh_cb(), triggered by rd_kafka_poll(), et.al. This callback will be triggered when it is time to refresh the client's OAUTHBEARER token. Also see rd_kafka_conf_enable_sasl_queue(). + OauthbearerTokenRefreshCb string `json:"oauthbearer_token_refresh_cb,omitempty" ` + // Set to "default" or "oidc" to control which login method to be used. If set to "oidc", the following properties must also be specified: sasl.oauthbearer.client.id, sasl.oauthbearer.client.secret, and sasl.oauthbearer.token.endpoint.url. + SaslOauthbearerMethod string `json:"sasl.oauthbearer.method,omitempty" ` + // Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when sasl.oauthbearer.method is set to "oidc". + SaslOauthbearerClientId string `json:"sasl.oauthbearer.client.id,omitempty" ` + // Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when sasl.oauthbearer.method is set to "oidc". + SaslOauthbearerClientSecret string `json:"sasl.oauthbearer.client.secret,omitempty" ` + // Client use this to specify the scope of the access request to the broker. Only used when sasl.oauthbearer.method is set to "oidc". + SaslOauthbearerScope string `json:"sasl.oauthbearer.scope,omitempty" ` + // Allow additional information to be provided to the broker. Comma-separated list of key=value pairs. E.g., "supportFeatureX=true,organizationId=sales-emea".Only used when sasl.oauthbearer.method is set to "oidc". + SaslOauthbearerExtensions string `json:"sasl.oauthbearer.extensions,omitempty" ` + // OAuth/OIDC issuer token endpoint HTTP(S) URI used to retrieve token. Only used when sasl.oauthbearer.method is set to "oidc". + SaslOauthbearerTokenEndpointUrl string `json:"sasl.oauthbearer.token.endpoint.url,omitempty" ` + // List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically. + PluginLibraryPaths string `json:"plugin.library.paths,omitempty" ` + // Interceptors added through rd_kafka_conf_interceptor_add_..() and any configuration handled by interceptors. + Interceptors string `json:"interceptors,omitempty"` +} + func (e *KafkaOutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "kafka2" if e.UseRdkafka { @@ -173,6 +363,13 @@ func (e *KafkaOutputConfig) ToDirective(secretLoader secret.SecretLoader, id str } else { kafka.SubDirectives = append(kafka.SubDirectives, buffer) } + if e.RdkafkaOptions != nil { + if rdkafkaOptions, err := e.RdkafkaOptions.ToDirective(secretLoader, id); err != nil { + return nil, err + } else { + kafka.SubDirectives = append(kafka.SubDirectives, rdkafkaOptions) + } + } if e.Format != nil { if format, err := e.Format.ToDirective(secretLoader, ""); err != nil { @@ -186,3 +383,9 @@ func (e *KafkaOutputConfig) ToDirective(secretLoader secret.SecretLoader, id str delete(kafka.Params, "use_rdkafka") return kafka, nil } + +func (o *RdkafkaOptions) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { + return types.NewFlatDirective(types.PluginMeta{ + Directive: "rdkafka_options", + }, o, secretLoader) +} diff --git a/pkg/sdk/logging/model/output/kafka_test.go b/pkg/sdk/logging/model/output/kafka_test.go index 288c0f01f..18a243e47 100644 --- a/pkg/sdk/logging/model/output/kafka_test.go +++ b/pkg/sdk/logging/model/output/kafka_test.go @@ -17,10 +17,11 @@ package output_test import ( "testing" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/render" "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/render" ) func TestKafka(t *testing.T) { @@ -69,6 +70,14 @@ brokers: kafka-headless.kafka.svc.cluster.local:29092 default_topic: topic use_rdkafka: true ssl_verify_hostname: false +rdkafka_options: + sasl.mechanisms: PLAIN + sasl.username: user + security.protocol: SASL_SSL + ssl.ca.location: /etc/ssl/certs/ca-certificates.crt + ssl.certificate.location: /etc/ssl/certs/tls.crt + ssl.key.location: /etc/ssl/certs/tls.key + ssl.key.password: password format: type: json buffer: @@ -91,6 +100,15 @@ buffer: timekey_use_utc true timekey_wait 30s + + sasl.mechanisms PLAIN + sasl.username user + security.protocol SASL_SSL + ssl.ca.location /etc/ssl/certs/ca-certificates.crt + ssl.certificate.location /etc/ssl/certs/tls.crt + ssl.key.location /etc/ssl/certs/tls.key + ssl.key.password password + @type json diff --git a/pkg/sdk/logging/model/output/zz_generated.deepcopy.go b/pkg/sdk/logging/model/output/zz_generated.deepcopy.go index cca345225..b5bac4221 100644 --- a/pkg/sdk/logging/model/output/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/model/output/zz_generated.deepcopy.go @@ -692,6 +692,11 @@ func (in *HTTPOutputConfig) DeepCopy() *HTTPOutputConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KafkaOutputConfig) DeepCopyInto(out *KafkaOutputConfig) { *out = *in + if in.RdkafkaOptions != nil { + in, out := &in.RdkafkaOptions, &out.RdkafkaOptions + *out = new(RdkafkaOptions) + **out = **in + } if in.Headers != nil { in, out := &in.Headers, &out.Headers *out = make(map[string]string, len(*in))