-
Notifications
You must be signed in to change notification settings - Fork 14.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
KAFKA-7747; Check for truncation after leader changes [KIP-320] (#6371)
After the client detects a leader change we need to check the offset of the current leader for truncation. These changes were part of KIP-320: https://cwiki.apache.org/confluence/display/KAFKA/KIP-320%3A+Allow+fetchers+to+detect+and+handle+log+truncation. Reviewers: Jason Gustafson <[email protected]>
- Loading branch information
Showing
23 changed files
with
1,574 additions
and
240 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
50 changes: 50 additions & 0 deletions
50
clients/src/main/java/org/apache/kafka/clients/consumer/LogTruncationException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kafka.clients.consumer; | ||
|
|
||
| import org.apache.kafka.common.TopicPartition; | ||
| import org.apache.kafka.common.utils.Utils; | ||
|
|
||
| import java.util.Collections; | ||
| import java.util.Map; | ||
| import java.util.function.Function; | ||
|
|
||
| /** | ||
| * In the even of unclean leader election, the log will be truncated, | ||
| * previously committed data will be lost, and new data will be written | ||
| * over these offsets. When this happens, the consumer will detect the | ||
| * truncation and raise this exception (if no automatic reset policy | ||
| * has been defined) with the first offset to diverge from what the | ||
| * consumer read. | ||
| */ | ||
| public class LogTruncationException extends OffsetOutOfRangeException { | ||
|
|
||
| private final Map<TopicPartition, OffsetAndMetadata> divergentOffsets; | ||
|
|
||
| public LogTruncationException(Map<TopicPartition, OffsetAndMetadata> divergentOffsets) { | ||
| super(Utils.transformMap(divergentOffsets, Function.identity(), OffsetAndMetadata::offset)); | ||
| this.divergentOffsets = Collections.unmodifiableMap(divergentOffsets); | ||
| } | ||
|
|
||
| /** | ||
| * Get the offsets for the partitions which were truncated. This is the first offset which is known to diverge | ||
| * from what the consumer read. | ||
| */ | ||
| public Map<TopicPartition, OffsetAndMetadata> divergentOffsets() { | ||
| return divergentOffsets; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
75 changes: 75 additions & 0 deletions
75
clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncClient.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kafka.clients.consumer.internals; | ||
|
|
||
| import org.apache.kafka.clients.ClientResponse; | ||
| import org.apache.kafka.common.Node; | ||
| import org.apache.kafka.common.requests.AbstractRequest; | ||
| import org.apache.kafka.common.requests.AbstractResponse; | ||
| import org.apache.kafka.common.utils.LogContext; | ||
| import org.slf4j.Logger; | ||
|
|
||
| public abstract class AsyncClient<T1, Req extends AbstractRequest, Resp extends AbstractResponse, T2> { | ||
|
|
||
| private final Logger log; | ||
| private final ConsumerNetworkClient client; | ||
|
|
||
| AsyncClient(ConsumerNetworkClient client, LogContext logContext) { | ||
| this.client = client; | ||
| this.log = logContext.logger(getClass()); | ||
| } | ||
|
|
||
| public RequestFuture<T2> sendAsyncRequest(Node node, T1 requestData) { | ||
| AbstractRequest.Builder<Req> requestBuilder = prepareRequest(node, requestData); | ||
|
|
||
| return client.send(node, requestBuilder).compose(new RequestFutureAdapter<ClientResponse, T2>() { | ||
| @Override | ||
| @SuppressWarnings("unchecked") | ||
| public void onSuccess(ClientResponse value, RequestFuture<T2> future) { | ||
| Resp resp; | ||
| try { | ||
| resp = (Resp) value.responseBody(); | ||
| } catch (ClassCastException cce) { | ||
| log.error("Could not cast response body", cce); | ||
| future.raise(cce); | ||
| return; | ||
| } | ||
| log.trace("Received {} {} from broker {}", resp.getClass().getSimpleName(), resp, node); | ||
| try { | ||
| future.complete(handleResponse(node, requestData, resp)); | ||
| } catch (RuntimeException e) { | ||
| if (!future.isDone()) { | ||
| future.raise(e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void onFailure(RuntimeException e, RequestFuture<T2> future1) { | ||
| future1.raise(e); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| protected Logger logger() { | ||
| return log; | ||
| } | ||
|
|
||
| protected abstract AbstractRequest.Builder<Req> prepareRequest(Node node, T1 requestData); | ||
|
|
||
| protected abstract T2 handleResponse(Node node, T1 requestData, Resp response); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.