-
Notifications
You must be signed in to change notification settings - Fork 107
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Allow to commit offset in case of deserialization errors (#761)
* Allow to commit offset in case of serialization errors * Disable test kafka listeners unless running their specific test * Add documentation * Update test to reflect current behavior Previous behavior was buggy. It was fixed by: #771 * Create config classes for the default kafka listener exception handler * Update documentation * Add an empty, `@Deprecated` constructor --------- Co-authored-by: Guillermo Calvo <[email protected]> Co-authored-by: Guillermo Calvo <[email protected]>
- Loading branch information
3 people
authored
Aug 18, 2023
1 parent
0e575af
commit 1281d47
Showing
5 changed files
with
332 additions
and
4 deletions.
There are no files selected for viewing
35 changes: 35 additions & 0 deletions
35
...cronaut/configuration/kafka/config/DefaultKafkaListenerExceptionHandlerConfiguration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright 2017-2023 original authors | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * https://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.micronaut.configuration.kafka.config; | ||
|
|
||
| import io.micronaut.core.util.Toggleable; | ||
|
|
||
| /** | ||
| * Default Kafka listener exception handler configuration. | ||
| * @since 5.1.0 | ||
| */ | ||
| public interface DefaultKafkaListenerExceptionHandlerConfiguration extends Toggleable { | ||
|
|
||
| /** | ||
| * @return Whether to skip record on deserialization failure. | ||
| */ | ||
| boolean isSkipRecordOnDeserializationFailure(); | ||
|
|
||
| /** | ||
| * @return Whether to commit record on deserialization failure. | ||
| */ | ||
| boolean isCommitRecordOnDeserializationFailure(); | ||
| } |
75 changes: 75 additions & 0 deletions
75
...nfiguration/kafka/config/DefaultKafkaListenerExceptionHandlerConfigurationProperties.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| /* | ||
| * Copyright 2017-2023 original authors | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * https://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.micronaut.configuration.kafka.config; | ||
|
|
||
| import io.micronaut.context.annotation.ConfigurationProperties; | ||
|
|
||
| /** | ||
| * {@link ConfigurationProperties} implementation of {@link DefaultKafkaListenerExceptionHandlerConfiguration}. | ||
| * @since 5.1.0 | ||
| */ | ||
| @ConfigurationProperties(DefaultKafkaListenerExceptionHandlerConfigurationProperties.PREFIX) | ||
| public class DefaultKafkaListenerExceptionHandlerConfigurationProperties implements DefaultKafkaListenerExceptionHandlerConfiguration { | ||
|
|
||
| /** | ||
| * The default prefix used for the default Kafka listener exception handler configuration. | ||
| */ | ||
| public static final String PREFIX = AbstractKafkaConfiguration.PREFIX + ".default-listener-exception-handler"; | ||
|
|
||
| /** | ||
| * The default value for {@code skipRecordOnDeserializationFailure}. | ||
| */ | ||
| @SuppressWarnings("WeakerAccess") | ||
| public static final boolean DEFAULT_SKIP_RECORD_ON_DESERIALIZATION_FAILURE = true; | ||
|
|
||
| /** | ||
| * The default value for {@code commitRecordOnDeserializationFailure}. | ||
| */ | ||
| @SuppressWarnings("WeakerAccess") | ||
| public static final boolean DEFAULT_COMMIT_RECORD_ON_DESERIALIZATION_FAILURE = false; | ||
|
|
||
| private boolean skipRecordOnDeserializationFailure = DEFAULT_SKIP_RECORD_ON_DESERIALIZATION_FAILURE; | ||
|
|
||
| private boolean commitRecordOnDeserializationFailure = DEFAULT_COMMIT_RECORD_ON_DESERIALIZATION_FAILURE; | ||
|
|
||
| @Override | ||
| public boolean isSkipRecordOnDeserializationFailure() { | ||
| return skipRecordOnDeserializationFailure; | ||
| } | ||
|
|
||
| /** | ||
| * Whether to skip record on deserialization failure. Default value {@value #DEFAULT_SKIP_RECORD_ON_DESERIALIZATION_FAILURE} | ||
| * | ||
| * @param skipRecordOnDeserializationFailure Whether to skip record on deserialization failure. | ||
| */ | ||
| public void setSkipRecordOnDeserializationFailure(boolean skipRecordOnDeserializationFailure) { | ||
| this.skipRecordOnDeserializationFailure = skipRecordOnDeserializationFailure; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isCommitRecordOnDeserializationFailure() { | ||
| return commitRecordOnDeserializationFailure; | ||
| } | ||
|
|
||
| /** | ||
| * Whether to commit record on deserialization failure. Default value {@value #DEFAULT_COMMIT_RECORD_ON_DESERIALIZATION_FAILURE} | ||
| * | ||
| * @param commitRecordOnDeserializationFailure Whether to commit record on deserialization failure. | ||
| */ | ||
| public void setCommitRecordOnDeserializationFailure(boolean commitRecordOnDeserializationFailure) { | ||
| this.commitRecordOnDeserializationFailure = commitRecordOnDeserializationFailure; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
166 changes: 166 additions & 0 deletions
166
.../micronaut/configuration/kafka/exceptions/DefaultKafkaListenerExceptionHandlerSpec.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,166 @@ | ||
| package io.micronaut.configuration.kafka.exceptions | ||
|
|
||
| import io.micronaut.configuration.kafka.config.DefaultKafkaListenerExceptionHandlerConfiguration | ||
| import io.micronaut.configuration.kafka.config.DefaultKafkaListenerExceptionHandlerConfigurationProperties | ||
| import io.micronaut.context.annotation.Property | ||
| import org.apache.kafka.clients.consumer.OffsetAndMetadata | ||
| import org.apache.kafka.clients.producer.ProducerConfig | ||
| import io.micronaut.configuration.kafka.AbstractEmbeddedServerSpec | ||
| import io.micronaut.configuration.kafka.annotation.KafkaClient | ||
| import io.micronaut.configuration.kafka.annotation.KafkaListener | ||
| import io.micronaut.configuration.kafka.annotation.Topic | ||
| import io.micronaut.context.annotation.Requires | ||
| import org.apache.kafka.common.TopicPartition | ||
|
|
||
| import static io.micronaut.configuration.kafka.annotation.OffsetReset.EARLIEST | ||
| import static io.micronaut.configuration.kafka.annotation.OffsetStrategy.DISABLED | ||
|
|
||
| class DefaultKafkaListenerExceptionHandlerSpec extends AbstractEmbeddedServerSpec { | ||
|
|
||
| private static final String TOPIC_SEEK = "on-deserialization-error-seek" | ||
| private static final String TOPIC_COMMIT = "on-deserialization-error-commit" | ||
| private static final String TOPIC_NOTHING = "on-deserialization-error-do-nothing" | ||
|
|
||
| void "test seek past record on deserialization error by default"() { | ||
| given: | ||
| StringProducer stringProducer = context.getBean(StringProducer) | ||
| DefaultBehaviorOnDeserializationErrorConsumer consumer = context.getBean(DefaultBehaviorOnDeserializationErrorConsumer) | ||
|
|
||
| when: "A producer sends a message with wrong serialization" | ||
| stringProducer.sendToSeekTopic("not-a-uuid") | ||
|
|
||
| then: "The message is skipped with a seek() but not committed" | ||
| conditions.eventually { | ||
| consumer.currentPosition == 1 | ||
| consumer.committedOffset == 0 | ||
| } | ||
| } | ||
|
|
||
| void "test commit record on deserialization error"() { | ||
| given: | ||
| StringProducer stringProducer = context.getBean(StringProducer) | ||
| CommitOnDeserializationErrorConsumer consumer = context.getBean(CommitOnDeserializationErrorConsumer) | ||
|
|
||
| when: "A producer sends a message with wrong serialization" | ||
| stringProducer.sendToCommitTopic("not-a-uuid") | ||
|
|
||
| then: "The message is skipped and committed" | ||
| conditions.eventually { | ||
| consumer.currentPosition == 1 | ||
| consumer.committedOffset == 1 | ||
| } | ||
| } | ||
|
|
||
| void "test do nothing on deserialization error"() { | ||
| given: | ||
| StringProducer stringProducer = context.getBean(StringProducer) | ||
| DoNothingOnDeserializationErrorConsumer consumer = context.getBean(DoNothingOnDeserializationErrorConsumer) | ||
|
|
||
| when: "A producer sends a message with wrong serialization" | ||
| stringProducer.sendToDoNothingTopic("not-a-uuid") | ||
|
|
||
| then: "The message is skipped, but not committed" | ||
| conditions.eventually { | ||
| consumer.currentPosition > 0 | ||
| consumer.committedOffset == 0 | ||
| } | ||
| } | ||
|
|
||
| static abstract class AbstractOnDeserializationErrorConsumer implements KafkaListenerExceptionHandler { | ||
| Long currentPosition = -1 | ||
| Long committedOffset = -1 | ||
| DefaultKafkaListenerExceptionHandlerConfiguration config = new DefaultKafkaListenerExceptionHandlerConfigurationProperties() | ||
| DefaultKafkaListenerExceptionHandler errorHandler = new DefaultKafkaListenerExceptionHandler(config) | ||
| String topic | ||
|
|
||
| AbstractOnDeserializationErrorConsumer(String topic) { | ||
| this.topic = topic | ||
| } | ||
|
|
||
| @Override | ||
| void handle(KafkaListenerException exception) { | ||
| errorHandler.handle(exception) | ||
| TopicPartition tp = new TopicPartition(topic, 0) | ||
| currentPosition = exception.kafkaConsumer.position(tp) | ||
| OffsetAndMetadata committedOffsetAndMetadata = exception.kafkaConsumer.committed(tp) | ||
| if (committedOffsetAndMetadata != null) { | ||
| committedOffset = committedOffsetAndMetadata.offset() | ||
| } else { | ||
| committedOffset = 0 | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Requires(property = 'spec.name', value = 'DefaultKafkaListenerExceptionHandlerSpec') | ||
| @KafkaListener( | ||
| offsetReset = EARLIEST, | ||
| offsetStrategy = DISABLED, | ||
| properties = [ | ||
| @Property(name = ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, | ||
| value = "org.apache.kafka.common.serialization.UUIDSerializer") | ||
| ] | ||
| ) | ||
| static class DefaultBehaviorOnDeserializationErrorConsumer extends AbstractOnDeserializationErrorConsumer { | ||
| DefaultBehaviorOnDeserializationErrorConsumer() { | ||
| super(TOPIC_SEEK) | ||
| } | ||
|
|
||
| @Topic(TOPIC_SEEK) | ||
| void receive(UUID uuid) { | ||
| } | ||
| } | ||
|
|
||
| @Requires(property = "spec.name", value = "DefaultKafkaListenerExceptionHandlerSpec") | ||
| @KafkaListener( | ||
| offsetReset = EARLIEST, | ||
| offsetStrategy = DISABLED, | ||
| properties = [ | ||
| @Property(name = ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, | ||
| value = "org.apache.kafka.common.serialization.UUIDSerializer") | ||
| ] | ||
| ) | ||
| static class CommitOnDeserializationErrorConsumer extends AbstractOnDeserializationErrorConsumer { | ||
| CommitOnDeserializationErrorConsumer() { | ||
| super(TOPIC_COMMIT) | ||
| errorHandler.setSkipRecordOnDeserializationFailure(true) | ||
| errorHandler.setCommitRecordOnDeserializationFailure(true) | ||
| } | ||
|
|
||
| @Topic(TOPIC_COMMIT) | ||
| void receive(UUID uuid) { | ||
| } | ||
| } | ||
|
|
||
| @Requires(property = "spec.name", value = "DefaultKafkaListenerExceptionHandlerSpec") | ||
| @KafkaListener( | ||
| offsetReset = EARLIEST, | ||
| offsetStrategy = DISABLED, | ||
| properties = [ | ||
| @Property(name = ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, | ||
| value = "org.apache.kafka.common.serialization.UUIDSerializer") | ||
| ] | ||
| ) | ||
| static class DoNothingOnDeserializationErrorConsumer extends AbstractOnDeserializationErrorConsumer { | ||
| DoNothingOnDeserializationErrorConsumer() { | ||
| super(TOPIC_NOTHING) | ||
| errorHandler.setSkipRecordOnDeserializationFailure(false) | ||
| } | ||
|
|
||
| @Topic(TOPIC_NOTHING) | ||
| void receive(UUID uuid) { | ||
| } | ||
| } | ||
|
|
||
| @Requires(property = 'spec.name', value = 'DefaultKafkaListenerExceptionHandlerSpec') | ||
| @KafkaClient | ||
| static interface StringProducer { | ||
| @Topic(TOPIC_SEEK) | ||
| void sendToSeekTopic(String message) | ||
|
|
||
| @Topic(TOPIC_COMMIT) | ||
| void sendToCommitTopic(String message) | ||
|
|
||
| @Topic(TOPIC_NOTHING) | ||
| void sendToDoNothingTopic(String message) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters