From 2c5a516e20349be22ae82ba5c7658b6ffc3b53ff Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 13 Sep 2022 17:34:05 +0800 Subject: [PATCH 01/18] Implement delayed message index bucket snapshot (create/load/recover) --- .../apache/pulsar/broker/delayed/Bucket.java | 122 ++++ .../delayed/BucketDelayedDeliveryTracker.java | 628 ++++++++++++++++++ .../InMemoryDelayedDeliveryTracker.java | 42 +- ...PersistentDispatcherMultipleConsumers.java | 4 + .../BuketDelayedDeliveryTrackerTest.java | 197 ++++++ .../delayed/InMemoryDeliveryTrackerTest.java | 326 ++++----- .../delayed/MockBucketSnapshotStorage.java | 159 +++++ .../broker/delayed/MockManagedCursor.java | 406 +++++++++++ 8 files changed, 1674 insertions(+), 210 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/Bucket.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/Bucket.java new file mode 100644 index 0000000000000..4a3933f618e84 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/Bucket.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; +import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELIMITER; +import com.google.protobuf.ByteString; +import java.util.BitSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; + +@Data +@AllArgsConstructor +public class Bucket { + + long startLedgerId; + long endLedgerId; + + Map delayedIndexBitMap; + + long numberBucketDelayedMessages; + + int lastSegmentEntryId; + + int currentSegmentEntryId; + + long snapshotLength; + + boolean active; + + volatile CompletableFuture snapshotCreateFuture; + + Bucket(long startLedgerId, long endLedgerId, Map delayedIndexBitMap) { + this(startLedgerId, endLedgerId, delayedIndexBitMap, -1, -1, 0, 0, true, null); + } + + long covertDelayIndexMapAndCount(int startSnapshotIndex, List segmentMetadata) { + delayedIndexBitMap.clear(); + MutableLong numberMessages = new MutableLong(0); + for (int i = startSnapshotIndex; i < segmentMetadata.size(); i++) { + Map bitByteStringMap = segmentMetadata.get(i).getDelayedIndexBitMapMap(); + bitByteStringMap.forEach((k, v) -> { + boolean exist = delayedIndexBitMap.containsKey(k); + byte[] bytes = v.toByteArray(); + BitSet bitSet = BitSet.valueOf(bytes); + numberMessages.add(bitSet.cardinality()); + if (!exist) { + delayedIndexBitMap.put(k, bitSet); + } else { + delayedIndexBitMap.get(k).or(bitSet); + } + }); + } + return numberMessages.longValue(); + } + + boolean containsMessage(long ledgerId, int entryId) { + if (delayedIndexBitMap == null) { + return false; + } + + BitSet bitSet = delayedIndexBitMap.get(ledgerId); + if (bitSet == null) { + return false; + } + return bitSet.get(entryId); + } + + void putIndexBit(long ledgerId, long entryId) { + if (entryId < Integer.MAX_VALUE) { + delayedIndexBitMap.compute(ledgerId, (k, v) -> new BitSet()).set((int) entryId); + } + } + + boolean removeIndexBit(long ledgerId, int entryId) { + if (delayedIndexBitMap == null) { + return false; + } + + boolean contained = false; + BitSet bitSet = delayedIndexBitMap.get(ledgerId); + if (bitSet != null && bitSet.get(entryId)) { + contained = true; + bitSet.clear(entryId); + + if (bitSet.isEmpty()) { + delayedIndexBitMap.remove(ledgerId); + } + + if (numberBucketDelayedMessages > 0) { + numberBucketDelayedMessages--; + } + } + return contained; + } + + public String bucketKey() { + return String.join(DELIMITER, DELAYED_BUCKET_KEY_PREFIX, String.valueOf(startLedgerId), + String.valueOf(endLedgerId)); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java new file mode 100644 index 0000000000000..0a04a29d5a4fc --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -0,0 +1,628 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; +import com.google.common.collect.Table; +import com.google.common.collect.TreeRangeMap; +import com.google.protobuf.ByteString; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import java.time.Clock; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; +import javax.annotation.concurrent.ThreadSafe; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; + +@Slf4j +@ThreadSafe +public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { + + public static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar_internal.delayed.bucket"; + + public static final String DELIMITER = "_"; + + private final long minIndexCountPerBucket; + + private final long timeStepPerBucketSnapshotSegment; + + private final int maxNumBuckets; + + private final ManagedCursor cursor; + + public final BucketSnapshotStorage bucketSnapshotStorage; + + private long numberDelayedMessages; + + private final Bucket lastMutableBucket; + + private final TripleLongPriorityQueue sharedBucketPriorityQueue; + + private final RangeMap immutableBuckets; + + private final Table snapshotSegmentLastIndexTable; + + BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, + Timer timer, long tickTimeMillis, + boolean isDelayedDeliveryDeliverAtTimeStrict, + BucketSnapshotStorage bucketSnapshotStorage, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + int maxNumBuckets) { + this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, + bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegment, maxNumBuckets); + } + + BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, + Timer timer, long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict, + BucketSnapshotStorage bucketSnapshotStorage, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + int maxNumBuckets) { + super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); + this.minIndexCountPerBucket = minIndexCountPerBucket; + this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; + this.maxNumBuckets = maxNumBuckets; + this.cursor = dispatcher.getCursor(); + this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); + this.immutableBuckets = TreeRangeMap.create(); + this.snapshotSegmentLastIndexTable = HashBasedTable.create(); + + this.bucketSnapshotStorage = bucketSnapshotStorage; + + numberDelayedMessages = recoverBucketSnapshot(); + + this.lastMutableBucket = new Bucket(-1L, -1L, new HashMap<>()); + } + + @SneakyThrows + private long recoverBucketSnapshot() { + List> completableFutures = new ArrayList<>(); + this.cursor.getCursorProperties().keySet().forEach(key -> { + if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) { + String[] keys = key.split(DELIMITER); + checkArgument(keys.length == 3); + Bucket bucket = createImmutableBucket(Long.parseLong(keys[1]), Long.parseLong(keys[2])); + completableFutures.add(asyncLoadNextBucketSnapshotEntry(bucket, true)); + } + }); + + if (completableFutures.isEmpty()) { + return 0; + } + + FutureUtil.waitForAll(completableFutures).get(); + + MutableLong numberDelayedMessages = new MutableLong(0); + immutableBuckets.asMapOfRanges().values().forEach(bucket -> { + numberDelayedMessages.add(bucket.numberBucketDelayedMessages); + }); + + log.info("[{}] Recover delayed message index bucket snapshot finish, buckets: {}, numberDelayedMessages: {}", + dispatcher.getName(), immutableBuckets.asMapOfRanges().size(), numberDelayedMessages.getValue()); + + return numberDelayedMessages.getValue(); + } + + private void moveScheduledMessageToSharedQueue(long cutoffTime) { + TripleLongPriorityQueue priorityQueue = getPriorityQueue(); + while (!priorityQueue.isEmpty()) { + long timestamp = priorityQueue.peekN1(); + if (timestamp > cutoffTime) { + break; + } + + long ledgerId = priorityQueue.peekN2(); + long entryId = priorityQueue.peekN3(); + sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); + + priorityQueue.pop(); + } + } + + @Override + public void run(Timeout timeout) throws Exception { + synchronized (this) { + moveScheduledMessageToSharedQueue(getCutoffTime()); + } + super.run(timeout); + } + + private Optional findBucket(long ledgerId) { + if (immutableBuckets.asMapOfRanges().isEmpty()) { + return Optional.empty(); + } + + Range span = immutableBuckets.span(); + if (!span.contains(ledgerId)) { + return Optional.empty(); + } + return Optional.ofNullable(immutableBuckets.get(ledgerId)); + } + + private Long getBucketIdByBucketKey(String bucketKey) { + String bucketIdStr = cursor.getCursorProperties().get(bucketKey); + if (StringUtils.isBlank(bucketIdStr)) { + return null; + } + return Long.valueOf(bucketIdStr); + } + + private Bucket createImmutableBucket(long startLedgerId, long endLedgerId) { + Bucket newBucket = new Bucket(startLedgerId, endLedgerId, new HashMap<>()); + immutableBuckets.put(Range.closed(startLedgerId, endLedgerId), newBucket); + return newBucket; + } + + private CompletableFuture asyncSaveBucketSnapshot( + final String bucketKey, SnapshotMetadata snapshotMetadata, + List bucketSnapshotSegments) { + Long bucketId = getBucketIdByBucketKey(bucketKey); + checkArgument(bucketId == null); + + return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments) + .thenCompose(newBucketId -> putBucketKeyId(bucketKey, newBucketId)); + } + + private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { + Objects.requireNonNull(bucketId); + return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), + ManagedLedgerException.BadVersionException.class).thenApply(__ -> bucketId); + } + + private CompletableFuture asyncCreateBucketSnapshot() { + TripleLongPriorityQueue priorityQueue = super.getPriorityQueue(); + if (priorityQueue.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + long numMessages = 0; + + final long startLedgerId = lastMutableBucket.startLedgerId; + final long endLedgerId = lastMutableBucket.endLedgerId; + + List bucketSnapshotSegments = new ArrayList<>(); + List segmentMetadataList = new ArrayList<>(); + Map bitMap = new HashMap<>(); + SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); + SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); + + long currentTimestampUpperLimit = 0; + while (!priorityQueue.isEmpty()) { + long timestamp = priorityQueue.peekN1(); + if (currentTimestampUpperLimit == 0) { + currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1; + } + + long ledgerId = priorityQueue.peekN2(); + long entryId = priorityQueue.peekN3(); + + checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); + + // Move first segment of bucket snapshot to sharedBucketPriorityQueue + if (segmentMetadataList.size() == 0) { + sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); + } + + priorityQueue.pop(); + numMessages++; + + DelayedIndex delayedIndex = DelayedIndex.newBuilder() + .setTimestamp(timestamp) + .setLedgerId(ledgerId) + .setEntryId(entryId).build(); + + if (entryId <= Integer.MAX_VALUE) { + bitMap.compute(ledgerId, (k, v) -> new BitSet()).set((int) entryId); + } + + snapshotSegmentBuilder.addIndexes(delayedIndex); + + if (priorityQueue.isEmpty() || priorityQueue.peekN1() > currentTimestampUpperLimit) { + segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); + currentTimestampUpperLimit = 0; + + Iterator> iterator = bitMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + ByteString byteString = ByteString.copyFrom(entry.getValue().toByteArray()); + segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), byteString); + iterator.remove(); + } + + segmentMetadataList.add(segmentMetadataBuilder.build()); + segmentMetadataBuilder.clear(); + + bucketSnapshotSegments.add(snapshotSegmentBuilder.build()); + snapshotSegmentBuilder.clear(); + } + } + + SnapshotMetadata bucketSnapshotMetadata = SnapshotMetadata.newBuilder() + .addAllMetadataList(segmentMetadataList) + .build(); + + final int lastSegmentEntryId = segmentMetadataList.size(); + + Bucket bucket = this.createImmutableBucket(startLedgerId, endLedgerId); + bucket.setCurrentSegmentEntryId(1); + bucket.setNumberBucketDelayedMessages(numMessages); + bucket.setLastSegmentEntryId(lastSegmentEntryId); + + // Add the first snapshot segment last message to snapshotSegmentLastMessageTable + checkArgument(!bucketSnapshotSegments.isEmpty()); + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); + DelayedIndex delayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); + snapshotSegmentLastIndexTable.put(delayedIndex.getLedgerId(), delayedIndex.getEntryId(), bucket); + + if (log.isDebugEnabled()) { + log.debug("[{}] Create bucket snapshot, bucket: {}", dispatcher.getName(), bucket); + } + + String bucketKey = bucket.bucketKey(); + CompletableFuture future = asyncSaveBucketSnapshot(bucketKey, + bucketSnapshotMetadata, bucketSnapshotSegments); + bucket.setSnapshotCreateFuture(future); + future.whenComplete((__, ex) -> { + if (ex == null) { + bucket.setSnapshotCreateFuture(null); + } else { + //TODO Record create snapshot failed + log.error("Failed to create snapshot: ", ex); + } + }); + + return future; + } + + + @SneakyThrows + private CompletableFuture asyncLoadNextBucketSnapshotEntry(Bucket bucket, boolean isRebuild) { + if (log.isDebugEnabled()) { + log.debug("[{}] Load next bucket snapshot data, bucket: {}", dispatcher.getName(), bucket); + } + if (bucket == null) { + return CompletableFuture.completedFuture(null); + } + + final CompletableFuture createFuture = bucket.snapshotCreateFuture; + if (createFuture != null) { + // Wait bucket snapshot create finish + createFuture.get(); + } + + final String bucketKey = bucket.bucketKey(); + final Long bucketId = getBucketIdByBucketKey(bucketKey); + Objects.requireNonNull(bucketId); + + CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); + if (isRebuild) { + final long cutoffTime = getCutoffTime(); + // Load Metadata of bucket snapshot + bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId).thenAccept(snapshotMetadata -> { + List metadataList = snapshotMetadata.getMetadataListList(); + + // Skip all already reach schedule time snapshot segments + int nextSnapshotEntryIndex = 0; + while (nextSnapshotEntryIndex < metadataList.size() + && metadataList.get(nextSnapshotEntryIndex).getMaxScheduleTimestamp() <= cutoffTime) { + nextSnapshotEntryIndex++; + } + + final int lastSegmentEntryId = metadataList.size(); + + long numberMessages = bucket.covertDelayIndexMapAndCount(nextSnapshotEntryIndex, metadataList); + bucket.setNumberBucketDelayedMessages(numberMessages); + bucket.setLastSegmentEntryId(lastSegmentEntryId); + + int nextSegmentEntryId = nextSnapshotEntryIndex + 1; + loadMetaDataFuture.complete(nextSegmentEntryId); + }); + } else { + loadMetaDataFuture.complete(bucket.currentSegmentEntryId + 1); + } + + CompletableFuture future = loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { + if (nextSegmentEntryId > bucket.lastSegmentEntryId) { + // TODO Delete bucket snapshot + return CompletableFuture.completedFuture(null); + } + + return bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId) + .thenAccept(bucketSnapshotSegments -> { + if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { + return; + } + + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); + List indexList = snapshotSegment.getIndexesList(); + DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); + + // Rebuild delayed message index bucket load data in parallel, so should be use synchronized + // to ensure data consistency + if (isRebuild) { + synchronized (snapshotSegmentLastIndexTable) { + this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), + lastDelayedIndex.getEntryId(), bucket); + } + + synchronized (sharedBucketPriorityQueue) { + for (DelayedIndex index : indexList) { + sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), + index.getEntryId()); + } + } + } else { + this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), + lastDelayedIndex.getEntryId(), bucket); + + for (DelayedIndex index : indexList) { + sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), + index.getEntryId()); + } + } + + bucket.setCurrentSegmentEntryId(nextSegmentEntryId); + }); + }); + return future; + } + + private void resetLastMutableBucketRange() { + lastMutableBucket.setStartLedgerId(-1L); + lastMutableBucket.setEndLedgerId(-1L); + } + + @Override + public synchronized boolean addMessage(long ledgerId, long entryId, long deliverAt) { + if (containsMessage(ledgerId, entryId)) { + messagesHaveFixedDelay = false; + return true; + } + + if (deliverAt < 0 || deliverAt <= getCutoffTime()) { + messagesHaveFixedDelay = false; + return false; + } + + boolean existBucket = findBucket(ledgerId).isPresent(); + + // Create bucket snapshot + if (ledgerId > lastMutableBucket.endLedgerId && !getPriorityQueue().isEmpty()) { + if (getPriorityQueue().size() >= minIndexCountPerBucket || existBucket) { + if (immutableBuckets.asMapOfRanges().size() >= maxNumBuckets) { + // TODO merge bucket snapshot (synchronize operate) + } + + asyncCreateBucketSnapshot(); + resetLastMutableBucketRange(); + } + } + + if (ledgerId < lastMutableBucket.startLedgerId || existBucket) { + // If (ledgerId < startLedgerId || existBucket) means that message index belong to previous bucket range, + // enter sharedBucketPriorityQueue directly + sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); + } else { + checkArgument(ledgerId >= lastMutableBucket.endLedgerId); + + getPriorityQueue().add(deliverAt, ledgerId, entryId); + + if (lastMutableBucket.startLedgerId == -1L) { + lastMutableBucket.setStartLedgerId(ledgerId); + } + lastMutableBucket.setEndLedgerId(ledgerId); + } + + // TODO If the bitSet is sparse, this memory cost very high to deduplication and skip read message + lastMutableBucket.putIndexBit(ledgerId, entryId); + numberDelayedMessages++; + + if (log.isDebugEnabled()) { + log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", dispatcher.getName(), ledgerId, entryId, + deliverAt - clock.millis()); + } + + updateTimer(); + + checkAndUpdateHighest(deliverAt); + + return true; + } + + @Override + public synchronized boolean hasMessageAvailable() { + long cutoffTime = getCutoffTime(); + + boolean hasMessageAvailable = !getPriorityQueue().isEmpty() && getPriorityQueue().peekN1() <= cutoffTime; + + hasMessageAvailable = hasMessageAvailable + || !sharedBucketPriorityQueue.isEmpty() && sharedBucketPriorityQueue.peekN1() <= cutoffTime; + if (!hasMessageAvailable) { + updateTimer(); + } + return hasMessageAvailable; + } + + @Override + protected long nextDeliveryTime() { + if (getPriorityQueue().isEmpty() && !sharedBucketPriorityQueue.isEmpty()) { + return sharedBucketPriorityQueue.peekN1(); + } else if (sharedBucketPriorityQueue.isEmpty() && !getPriorityQueue().isEmpty()) { + return getPriorityQueue().peekN1(); + } + long timestamp = getPriorityQueue().peekN1(); + long bucketTimestamp = sharedBucketPriorityQueue.peekN1(); + return Math.min(timestamp, bucketTimestamp); + } + + @Override + public synchronized long getNumberOfDelayedMessages() { + return numberDelayedMessages; + } + + @Override + public synchronized long getBufferMemoryUsage() { + return getPriorityQueue().bytesCapacity() + sharedBucketPriorityQueue.bytesCapacity(); + } + + @Override + @SneakyThrows + public synchronized Set getScheduledMessages(int maxMessages) { + long cutoffTime = getCutoffTime(); + + moveScheduledMessageToSharedQueue(cutoffTime); + + Set positions = new TreeSet<>(); + int n = maxMessages; + + while (n > 0 && !sharedBucketPriorityQueue.isEmpty()) { + long timestamp = sharedBucketPriorityQueue.peekN1(); + if (timestamp > cutoffTime) { + break; + } + + long ledgerId = sharedBucketPriorityQueue.peekN2(); + long entryId = sharedBucketPriorityQueue.peekN3(); + positions.add(new PositionImpl(ledgerId, entryId)); + + sharedBucketPriorityQueue.pop(); + removeIndexBit(ledgerId, entryId); + + Bucket bucket = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); + if (bucket != null && bucket.active) { + if (log.isDebugEnabled()) { + log.debug("[{}] Load next snapshot segment, bucket: {}", dispatcher.getName(), bucket); + } + // All message of current snapshot segment are scheduled, load next snapshot segment + asyncLoadNextBucketSnapshotEntry(bucket, false).get(); + } + + --n; + --numberDelayedMessages; + } + + if (numberDelayedMessages <= 0) { + // Reset to initial state + highestDeliveryTimeTracked = 0; + messagesHaveFixedDelay = true; + } + + updateTimer(); + + return positions; + } + + @Override + @SneakyThrows + public synchronized void clear() { + super.clear(); + cleanImmutableBuckets(true); + sharedBucketPriorityQueue.clear(); + resetLastMutableBucketRange(); + lastMutableBucket.delayedIndexBitMap.clear(); + snapshotSegmentLastIndexTable.clear(); + numberDelayedMessages = 0; + } + + @Override + @SneakyThrows + public synchronized void close() { + super.close(); + cleanImmutableBuckets(false); + lastMutableBucket.delayedIndexBitMap.clear(); + sharedBucketPriorityQueue.close(); + } + + private void cleanImmutableBuckets(boolean delete) { + if (immutableBuckets != null) { + Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); + while (iterator.hasNext()) { + Bucket bucket = iterator.next(); + if (bucket.delayedIndexBitMap != null) { + bucket.delayedIndexBitMap.clear(); + } + CompletableFuture snapshotGenerateFuture = bucket.snapshotCreateFuture; + if (snapshotGenerateFuture != null) { + if (delete) { + snapshotGenerateFuture.cancel(true); + // TODO delete bucket snapshot + } else { + try { + snapshotGenerateFuture.get(); + } catch (Exception e) { + log.warn("Failed wait to snapshot generate, bucket: {}", bucket); + } + } + } + iterator.remove(); + } + } + } + + private boolean removeIndexBit(long ledgerId, long entryId) { + if (entryId > Integer.MAX_VALUE) { + return false; + } + + if (lastMutableBucket.removeIndexBit(ledgerId, (int) entryId)) { + return true; + } + + return findBucket(ledgerId).map(bucket -> bucket.removeIndexBit(ledgerId, (int) entryId)).orElse(false); + } + + @Override + public boolean containsMessage(long ledgerId, long entryId) { + if (entryId > Integer.MAX_VALUE) { + return false; + } + + if (lastMutableBucket.containsMessage(ledgerId, (int) entryId)) { + return true; + } + + return findBucket(ledgerId).map(bucket -> bucket.containsMessage(ledgerId, (int) entryId)).orElse(false); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index da28ff19234b4..ef8fc2abf2bec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -35,7 +35,7 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T protected final TripleLongPriorityQueue priorityQueue = new TripleLongPriorityQueue(); - private final PersistentDispatcherMultipleConsumers dispatcher; + protected final PersistentDispatcherMultipleConsumers dispatcher; // Reference to the shared (per-broker) timer for delayed delivery private final Timer timer; @@ -49,9 +49,9 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T // Last time the TimerTask was triggered for this class private long lastTickRun; - private long tickTimeMillis; + protected long tickTimeMillis; - private final Clock clock; + protected final Clock clock; private final boolean isDelayedDeliveryDeliverAtTimeStrict; @@ -64,10 +64,10 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T // This is the timestamp of the message with the highest delivery time // If new added messages are lower than this, it means the delivery is requested // to be out-of-order. It gets reset to 0, once the tracker is emptied. - private long highestDeliveryTimeTracked = 0; + protected long highestDeliveryTimeTracked = 0; // Track whether we have seen all messages with fixed delay so far. - private boolean messagesHaveFixedDelay = true; + protected boolean messagesHaveFixedDelay = true; InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -99,7 +99,7 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T * * @return the cutoff time to determine whether a message is ready to deliver to the consumer */ - private long getCutoffTime() { + protected long getCutoffTime() { return isDelayedDeliveryDeliverAtTimeStrict ? clock.millis() : clock.millis() + tickTimeMillis; } @@ -119,15 +119,21 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { priorityQueue.add(deliverAt, ledgerId, entryId); updateTimer(); - // Check that new delivery time comes after the current highest, or at - // least within a single tick time interval of 1 second. + checkAndUpdateHighest(deliverAt); + + return true; + } + + /** + * Check that new delivery time comes after the current highest, or at + * least within a single tick time interval of 1 second. + */ + protected void checkAndUpdateHighest(long deliverAt) { if (deliverAt < (highestDeliveryTimeTracked - tickTimeMillis)) { messagesHaveFixedDelay = false; } highestDeliveryTimeTracked = Math.max(highestDeliveryTimeTracked, deliverAt); - - return true; } /** @@ -213,8 +219,8 @@ public long getBufferMemoryUsage() { * the last tick time plus the tickTimeMillis (to ensure we do not schedule the task more frequently than the * tickTimeMillis). */ - private void updateTimer() { - if (priorityQueue.isEmpty()) { + protected void updateTimer() { + if (getNumberOfDelayedMessages() == 0) { if (timeout != null) { currentTimeoutTarget = -1; timeout.cancel(); @@ -222,8 +228,7 @@ private void updateTimer() { } return; } - - long timestamp = priorityQueue.peekN1(); + long timestamp = nextDeliveryTime(); if (timestamp == currentTimeoutTarget) { // The timer is already set to the correct target time return; @@ -291,7 +296,7 @@ public boolean shouldPauseAllDeliveries() { // Pause deliveries if we know all delays are fixed within the lookahead window return fixedDelayDetectionLookahead > 0 && messagesHaveFixedDelay - && priorityQueue.size() >= fixedDelayDetectionLookahead + && getNumberOfDelayedMessages() >= fixedDelayDetectionLookahead && !hasMessageAvailable(); } @@ -299,4 +304,11 @@ public boolean shouldPauseAllDeliveries() { public boolean containsMessage(long ledgerId, long entryId) { return false; } + + protected TripleLongPriorityQueue getPriorityQueue() { + return priorityQueue; + } + protected long nextDeliveryTime() { + return priorityQueue.peekN1(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index b777087ba2f39..193ee07561d2d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -1120,6 +1120,10 @@ public long getDelayedTrackerMemoryUsage() { return 0; } + public ManagedCursor getCursor() { + return cursor; + } + protected int getStickyKeyHash(Entry entry) { return StickyKeyConsumerSelector.makeStickyKeyHash(peekStickyKey(entry.getDataBuffer())); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java new file mode 100644 index 0000000000000..84273b69b4231 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.lang.reflect.Method; +import java.time.Clock; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class BuketDelayedDeliveryTrackerTest extends InMemoryDeliveryTrackerTest { + + private final Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-bucket-delayed-delivery-test"), + 500, TimeUnit.MILLISECONDS); + + private BucketSnapshotStorage bucketSnapshotStorage; + + @AfterMethod + public void clean() throws Exception { + if (bucketSnapshotStorage != null) { + bucketSnapshotStorage.close(); + } + } + + @DataProvider(name = "delayedTracker") + @Override + public Object[][] provider(Method method) throws Exception { + dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + clock = mock(Clock.class); + clockTime = new AtomicLong(); + when(clock.millis()).then(x -> clockTime.get()); + + bucketSnapshotStorage = new MockBucketSnapshotStorage(); + bucketSnapshotStorage.start(); + ManagedCursor cursor = new MockManagedCursor("my_test_cursor"); + doReturn(cursor).when(dispatcher).getCursor(); + + final String methodName = method.getName(); + return switch (methodName) { + case "test" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, + false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + case "testWithTimer" -> { + Timer timer = mock(Timer.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + NavigableMap tasks = new TreeMap<>(); + + when(timer.newTimeout(any(), anyLong(), any())).then(invocation -> { + TimerTask task = invocation.getArgument(0, TimerTask.class); + long timeout = invocation.getArgument(1, Long.class); + TimeUnit unit = invocation.getArgument(2, TimeUnit.class); + long scheduleAt = clockTime.get() + unit.toMillis(timeout); + tasks.put(scheduleAt, task); + + Timeout t = mock(Timeout.class); + when(t.cancel()).then(i -> { + tasks.remove(scheduleAt, task); + return null; + }); + return t; + }); + + yield new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, + false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50), + tasks + }}; + } + case "testAddWithinTickTime" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock, + false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + case "testAddMessageWithStrictDelay" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + case "testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + case "testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict", "testRecoverSnapshot" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict", "testExistDelayedMessage" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + default -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), 50) + }}; + }; + } + + @Test(dataProvider = "delayedTracker") + public void testContainsMessage(DelayedDeliveryTracker tracker) { + tracker.addMessage(1, 1, 10); + tracker.addMessage(2, 2, 20); + + assertTrue(tracker.containsMessage(1, 1)); + clockTime.set(20); + + Set scheduledMessages = tracker.getScheduledMessages(1); + assertEquals(scheduledMessages.stream().findFirst().get().getEntryId(), 1); + + tracker.addMessage(3, 3, 30); + + tracker.addMessage(4, 4, 30); + + tracker.addMessage(5, 5, 30); + + tracker.addMessage(6, 6, 30); + + assertTrue(tracker.containsMessage(3, 3)); + + tracker.close(); + } + + @Test(dataProvider = "delayedTracker") + public void testRecoverSnapshot(DelayedDeliveryTracker tracker) throws Exception { + for (int i = 1; i <= 100; i++) { + tracker.addMessage(i, i, i * 10); + } + + assertEquals(tracker.getNumberOfDelayedMessages(), 100); + + clockTime.set(1 * 10); + + assertTrue(tracker.hasMessageAvailable()); + Set scheduledMessages = tracker.getScheduledMessages(100); + + assertEquals(scheduledMessages.size(), 1); + + tracker.addMessage(101, 101, 101 * 10); + + tracker.close(); + + clockTime.set(30 * 10); + + tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50); + + assertFalse(tracker.containsMessage(101, 101)); + assertEquals(tracker.getNumberOfDelayedMessages(), 70); + + clockTime.set(100 * 10); + + assertTrue(tracker.hasMessageAvailable()); + scheduledMessages = tracker.getScheduledMessages(70); + + assertEquals(scheduledMessages.size(), 70); + tracker.close(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 1a98233c3859a..8c8e6271f2797 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,21 +20,21 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.atMostOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; + import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; import io.netty.util.concurrent.DefaultThreadFactory; +import java.lang.reflect.Method; import java.time.Clock; import java.util.Collections; import java.util.NavigableMap; @@ -42,11 +42,11 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import lombok.Cleanup; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker") @@ -55,31 +55,99 @@ public class InMemoryDeliveryTrackerTest { // Create a single shared timer for the test. private final Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-in-memory-delayed-delivery-test"), 500, TimeUnit.MILLISECONDS); + protected PersistentDispatcherMultipleConsumers dispatcher; + protected Clock clock; + + protected AtomicLong clockTime; @AfterClass(alwaysRun = true) public void cleanup() { timer.stop(); } - @Test - public void test() throws Exception { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); + @DataProvider(name = "delayedTracker") + public Object[][] provider(Method method) throws Exception { + dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + clock = mock(Clock.class); + clockTime = new AtomicLong(); when(clock.millis()).then(x -> clockTime.get()); - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false, 0); + final String methodName = method.getName(); + return switch (methodName) { + case "test" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + false, 0) + }}; + case "testWithTimer" -> { + Timer timer = mock(Timer.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + NavigableMap tasks = new TreeMap<>(); + + when(timer.newTimeout(any(), anyLong(), any())).then(invocation -> { + TimerTask task = invocation.getArgument(0, TimerTask.class); + long timeout = invocation.getArgument(1, Long.class); + TimeUnit unit = invocation.getArgument(2, TimeUnit.class); + long scheduleAt = clockTime.get() + unit.toMillis(timeout); + tasks.put(scheduleAt, task); + + Timeout t = mock(Timeout.class); + when(t.cancel()).then(i -> { + tasks.remove(scheduleAt, task); + return null; + }); + return t; + }); + + yield new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + false, 0), + tasks + }}; + } + case "testAddWithinTickTime" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, + false, 0) + }}; + case "testAddMessageWithStrictDelay" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, + true, 0) + }}; + case "testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, 0) + }}; + case "testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100000, clock, + true, 0) + }}; + case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 500, clock, + true, 0) + }}; + case "testWithFixedDelays", "testWithMixedDelays","testWithNoDelays" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 500, clock, + true, 100) + }}; + default -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true, 0) + }}; + }; + } + @Test(dataProvider = "delayedTracker") + public void test(DelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); - assertTrue(tracker.addMessage(2, 2, 20)); - assertTrue(tracker.addMessage(1, 1, 10)); + assertTrue(tracker.addMessage(1, 2, 20)); + assertTrue(tracker.addMessage(2, 1, 10)); assertTrue(tracker.addMessage(3, 3, 30)); - assertTrue(tracker.addMessage(5, 5, 50)); - assertTrue(tracker.addMessage(4, 4, 40)); + assertTrue(tracker.addMessage(4, 5, 50)); + assertTrue(tracker.addMessage(5, 4, 40)); assertFalse(tracker.hasMessageAvailable()); assertEquals(tracker.getNumberOfDelayedMessages(), 5); @@ -113,38 +181,12 @@ public void test() throws Exception { assertEquals(tracker.getNumberOfDelayedMessages(), 0); assertFalse(tracker.hasMessageAvailable()); assertEquals(tracker.getScheduledMessages(10), Collections.emptySet()); - } - @Test - public void testWithTimer() throws Exception { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - Timer timer = mock(Timer.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - NavigableMap tasks = new TreeMap<>(); - - when(timer.newTimeout(any(), anyLong(), any())).then(invocation -> { - TimerTask task = invocation.getArgument(0, TimerTask.class); - long timeout = invocation.getArgument(1, Long.class); - TimeUnit unit = invocation.getArgument(2, TimeUnit.class); - long scheduleAt = clockTime.get() + unit.toMillis(timeout); - tasks.put(scheduleAt, task); - - Timeout t = mock(Timeout.class); - when(t.cancel()).then(i -> { - tasks.remove(scheduleAt, task); - return null; - }); - return t; - }); - - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false, 0); + tracker.close(); + } + @Test(dataProvider = "delayedTracker") + public void testWithTimer(DelayedDeliveryTracker tracker, NavigableMap tasks) throws Exception { assertTrue(tasks.isEmpty()); assertTrue(tracker.addMessage(2, 2, 20)); assertEquals(tasks.size(), 1); @@ -164,28 +206,20 @@ public void testWithTimer() throws Exception { Timeout cancelledTimeout = mock(Timeout.class); when(cancelledTimeout.isCancelled()).thenReturn(true); task.run(cancelledTimeout); - verifyZeroInteractions(dispatcher); + verify(dispatcher, atMostOnce()).readMoreEntries(); task.run(mock(Timeout.class)); verify(dispatcher).readMoreEntries(); + + tracker.close(); } /** * Adding a message that is about to expire within the tick time should lead * to a rejection from the tracker when isDelayedDeliveryDeliverAtTimeStrict is false. */ - @Test - public void testAddWithinTickTime() { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, - false, 0); - + @Test(dataProvider = "delayedTracker") + public void testAddWithinTickTime(DelayedDeliveryTracker tracker) { clockTime.set(0); assertFalse(tracker.addMessage(1, 1, 10)); @@ -195,19 +229,12 @@ public void testAddWithinTickTime() { assertTrue(tracker.addMessage(5, 5, 200)); assertEquals(tracker.getNumberOfDelayedMessages(), 2); - } - - public void testAddMessageWithStrictDelay() { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, - true, 0); + tracker.close(); + } + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithStrictDelay(DelayedDeliveryTracker tracker) { clockTime.set(10); // Verify behavior for the less than, equal to, and greater than deliverAt times. @@ -217,29 +244,22 @@ public void testAddMessageWithStrictDelay() { assertEquals(tracker.getNumberOfDelayedMessages(), 1); assertFalse(tracker.hasMessageAvailable()); + + tracker.close(); } /** * In this test, the deliverAt time is after now, but the deliverAt time is too early to run another tick, so the * tickTimeMillis determines the delay. */ - public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict() throws Exception { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 1000, clock, true, 0); - + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict(DelayedDeliveryTracker tracker) + throws Exception { // Set clock time, then run tracker to inherit clock time as the last tick time. clockTime.set(10000); Timeout timeout = mock(Timeout.class); when(timeout.isCancelled()).then(x -> false); - tracker.run(timeout); + ((InMemoryDelayedDeliveryTracker) tracker).run(timeout); verify(dispatcher, times(1)).readMoreEntries(); // Add a message that has a delivery time just after the previous run. It will get delivered based on the @@ -254,25 +274,17 @@ public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithSt // Not wait for the message delivery to get triggered. Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + + tracker.close(); } /** * In this test, the deliverAt time is after now, but before the (tickTimeMillis + now). Because there wasn't a * recent tick run, the deliverAt time determines the delay. */ - public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict() { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - // Use a large tick time to show that the message will get delivered earlier because there wasn't - // a previous tick run. - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 100000, clock, true, 0); - + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict( + DelayedDeliveryTracker tracker) { clockTime.set(500000); assertTrue(tracker.addMessage(1, 1, 500005)); @@ -281,23 +293,16 @@ public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStr // should get scheduled early when the tick duration has passed since the last tick. Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + + tracker.close(); } /** * In this test, the deliverAt time is after now plus tickTimeMillis, so the tickTimeMillis determines the delay. */ - public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict() throws Exception { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 500, clock, true, 0); - + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict(DelayedDeliveryTracker tracker) + throws Exception { clockTime.set(0); assertTrue(tracker.addMessage(1, 1, 2000)); @@ -305,27 +310,17 @@ public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict() throws // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has // passed where it would have been triggered if the tick time was doing the triggering. Thread.sleep(1000); - verifyNoInteractions(dispatcher); + verify(dispatcher).getCursor(); // Not wait for the message delivery to get triggered. Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> verify(dispatcher).readMoreEntries()); - } - - @Test - public void testWithFixedDelays() throws Exception { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - final long fixedDelayLookahead = 100; - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true, fixedDelayLookahead); + tracker.close(); + } + @Test(dataProvider = "delayedTracker") + public void testWithFixedDelays(DelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); assertTrue(tracker.addMessage(1, 1, 10)); @@ -347,6 +342,7 @@ public void testWithFixedDelays() throws Exception { clockTime.set(fixedDelayLookahead * 10); tracker.getScheduledMessages(100); + assertFalse(tracker.shouldPauseAllDeliveries()); // Empty the tracker @@ -356,22 +352,12 @@ public void testWithFixedDelays() throws Exception { } while (removed > 0); assertFalse(tracker.shouldPauseAllDeliveries()); - } - - @Test - public void testWithMixedDelays() throws Exception { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - long fixedDelayLookahead = 100; - - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true, fixedDelayLookahead); + tracker.close(); + } + @Test(dataProvider = "delayedTracker") + public void testWithMixedDelays(DelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); assertTrue(tracker.addMessage(1, 1, 10)); @@ -389,25 +375,15 @@ public void testWithMixedDelays() throws Exception { assertTrue(tracker.shouldPauseAllDeliveries()); // Add message with earlier delivery time - assertTrue(tracker.addMessage(5, 5, 5)); + assertTrue(tracker.addMessage(5, 6, 5)); assertFalse(tracker.shouldPauseAllDeliveries()); - } - - @Test - public void testWithNoDelays() throws Exception { - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - long fixedDelayLookahead = 100; - @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true, fixedDelayLookahead); + tracker.close(); + } + @Test(dataProvider = "delayedTracker") + public void testWithNoDelays(DelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); assertTrue(tracker.addMessage(1, 1, 10)); @@ -425,51 +401,11 @@ public void testWithNoDelays() throws Exception { assertTrue(tracker.shouldPauseAllDeliveries()); // Add message with no-delay - assertFalse(tracker.addMessage(5, 5, -1L)); + assertFalse(tracker.addMessage(5, 6, -1L)); assertFalse(tracker.shouldPauseAllDeliveries()); - } - - @Test - public void testClose() throws Exception { - Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-in-memory-delayed-delivery-test"), - 1, TimeUnit.MILLISECONDS); - - PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - - AtomicLong clockTime = new AtomicLong(); - Clock clock = mock(Clock.class); - when(clock.millis()).then(x -> clockTime.get()); - - final Exception[] exceptions = new Exception[1]; - - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true, 0) { - @Override - public void run(Timeout timeout) throws Exception { - super.timeout = timer.newTimeout(this, 1, TimeUnit.MILLISECONDS); - if (timeout == null || timeout.isCancelled()) { - return; - } - try { - this.priorityQueue.peekN1(); - } catch (Exception e) { - e.printStackTrace(); - exceptions[0] = e; - } - } - }; - - tracker.addMessage(1, 1, 10); - clockTime.set(10); - - Thread.sleep(300); tracker.close(); - - assertNull(exceptions[0]); - - timer.stop(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java new file mode 100644 index 0000000000000..352fea65be909 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; + +@Slf4j +public class MockBucketSnapshotStorage implements BucketSnapshotStorage { + + private final AtomicLong maxBucketId; + + private final Map> bucketSnapshots; + + private final ExecutorService executorService = + new ThreadPoolExecutor(10, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), + new DefaultThreadFactory("bucket-snapshot-storage-io")); + + public MockBucketSnapshotStorage() { + this.bucketSnapshots = new ConcurrentHashMap<>(); + this.maxBucketId = new AtomicLong(); + } + + @Override + public CompletableFuture createBucketSnapshot( + SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments) { + return CompletableFuture.supplyAsync(() -> { + long bucketId = maxBucketId.getAndIncrement(); + List entries = new ArrayList<>(); + byte[] bytes = snapshotMetadata.toByteArray(); + ByteBuf byteBuf = PooledByteBufAllocator.DEFAULT.directBuffer(bytes.length); + byteBuf.writeBytes(bytes); + entries.add(byteBuf); + this.bucketSnapshots.put(bucketId, entries); + return bucketId; + }, executorService).thenApply(bucketId -> { + List bufList = new ArrayList<>(); + for (SnapshotSegment snapshotSegment : bucketSnapshotSegments) { + byte[] bytes = snapshotSegment.toByteArray(); + ByteBuf byteBuf = PooledByteBufAllocator.DEFAULT.directBuffer(bytes.length); + byteBuf.writeBytes(bytes); + bufList.add(byteBuf); + } + bucketSnapshots.get(bucketId).addAll(bufList); + + return bucketId; + }); + } + + @Override + public CompletableFuture getBucketSnapshotMetadata(long bucketId) { + return CompletableFuture.supplyAsync(() -> { + ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(0); + SnapshotMetadata snapshotMetadata; + try { + snapshotMetadata = SnapshotMetadata.parseFrom(byteBuf.nioBuffer()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + return snapshotMetadata; + }, executorService); + } + + @Override + public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, + long lastSegmentEntryId) { + return CompletableFuture.supplyAsync(() -> { + List snapshotSegments = new ArrayList<>(); + long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size()); + for (int i = (int) firstSegmentEntryId; i <= lastEntryId ; i++) { + ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(i); + SnapshotSegment snapshotSegment; + try { + snapshotSegment = SnapshotSegment.parseFrom(byteBuf.nioBuffer()); + snapshotSegments.add(snapshotSegment); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + return snapshotSegments; + }, executorService); + } + + @Override + public CompletableFuture deleteBucketSnapshot(long bucketId) { + return CompletableFuture.supplyAsync(() -> { + List remove = this.bucketSnapshots.remove(bucketId); + if (remove != null) { + for (ByteBuf byteBuf : remove) { + byteBuf.release(); + } + } + return null; + }, executorService); + } + + @Override + public CompletableFuture getBucketSnapshotLength(long bucketId) { + return CompletableFuture.supplyAsync(() -> { + long length = 0; + List bufList = this.bucketSnapshots.get(bucketId); + for (ByteBuf byteBuf : bufList) { + length += byteBuf.array().length; + } + return length; + }, executorService); + } + + @Override + public void start() throws Exception { + + } + + @Override + public void close() throws Exception { + clean(); + } + + public void clean() { + for (List value : bucketSnapshots.values()) { + for (ByteBuf byteBuf : value) { + byteBuf.release(); + } + } + bucketSnapshots.clear(); + executorService.shutdownNow(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java new file mode 100644 index 0000000000000..51b0aee748f9b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -0,0 +1,406 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import com.google.common.base.Predicate; +import com.google.common.collect.Range; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedCursorMXBean; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; + +public class MockManagedCursor implements ManagedCursor { + + private final String name; + + private Map cursorProperties; + + public MockManagedCursor(String name) { + this.name = name; + this.cursorProperties = new ConcurrentHashMap<>(); + } + + @Override + public String getName() { + return null; + } + + @Override + public long getLastActive() { + return 0; + } + + @Override + public void updateLastActive() { + + } + + @Override + public Map getProperties() { + return null; + } + + @Override + public Map getCursorProperties() { + return this.cursorProperties; + } + + @Override + public CompletableFuture putCursorProperty(String key, String value) { + cursorProperties.put(key, value); + return CompletableFuture.completedFuture(null); + } + + public CompletableFuture removeCursorProperty(String key) { + cursorProperties.remove(key); + return CompletableFuture.completedFuture(null); + } + + @Override + public boolean putProperty(String key, Long value) { + return false; + } + + @Override + public boolean removeProperty(String key) { + return false; + } + + @Override + public List readEntries(int numberOfEntriesToRead) throws InterruptedException, ManagedLedgerException { + return null; + } + + @Override + public void asyncReadEntries(int numberOfEntriesToRead, AsyncCallbacks.ReadEntriesCallback callback, Object ctx, + PositionImpl maxPosition) { + + } + + @Override + public void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, + AsyncCallbacks.ReadEntriesCallback callback, Object ctx, PositionImpl maxPosition) { + + } + + @Override + public Entry getNthEntry(int n, IndividualDeletedEntries deletedEntries) + throws InterruptedException, ManagedLedgerException { + return null; + } + + @Override + public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, + AsyncCallbacks.ReadEntryCallback callback, Object ctx) { + + } + + @Override + public List readEntriesOrWait(int numberOfEntriesToRead) + throws InterruptedException, ManagedLedgerException { + return null; + } + + @Override + public List readEntriesOrWait(int maxEntries, long maxSizeBytes) + throws InterruptedException, ManagedLedgerException { + return null; + } + + @Override + public void asyncReadEntriesOrWait(int numberOfEntriesToRead, AsyncCallbacks.ReadEntriesCallback callback, + Object ctx, PositionImpl maxPosition) { + + } + + @Override + public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, AsyncCallbacks.ReadEntriesCallback callback, + Object ctx, PositionImpl maxPosition) { + + } + + @Override + public boolean cancelPendingReadRequest() { + return false; + } + + @Override + public boolean hasMoreEntries() { + return false; + } + + @Override + public long getNumberOfEntries() { + return 0; + } + + @Override + public long getNumberOfEntriesInBacklog(boolean isPrecise) { + return 0; + } + + @Override + public void markDelete(Position position) throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void markDelete(Position position, Map properties) + throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void asyncMarkDelete(Position position, AsyncCallbacks.MarkDeleteCallback callback, Object ctx) { + + } + + @Override + public void asyncMarkDelete(Position position, Map properties, + AsyncCallbacks.MarkDeleteCallback callback, Object ctx) { + + } + + @Override + public void delete(Position position) throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void asyncDelete(Position position, AsyncCallbacks.DeleteCallback callback, Object ctx) { + + } + + @Override + public void delete(Iterable positions) throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void asyncDelete(Iterable position, AsyncCallbacks.DeleteCallback callback, Object ctx) { + + } + + @Override + public Position getReadPosition() { + return null; + } + + @Override + public Position getMarkDeletedPosition() { + return null; + } + + @Override + public Position getPersistentMarkDeletedPosition() { + return null; + } + + @Override + public void rewind() { + + } + + @Override + public void seek(Position newReadPosition, boolean force) { + + } + + @Override + public void clearBacklog() throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void asyncClearBacklog(AsyncCallbacks.ClearBacklogCallback callback, Object ctx) { + + } + + @Override + public void skipEntries(int numEntriesToSkip, IndividualDeletedEntries deletedEntries) + throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void asyncSkipEntries(int numEntriesToSkip, IndividualDeletedEntries deletedEntries, + AsyncCallbacks.SkipEntriesCallback callback, Object ctx) { + + } + + @Override + public Position findNewestMatching(Predicate condition) throws InterruptedException, ManagedLedgerException { + return null; + } + + @Override + public Position findNewestMatching(FindPositionConstraint constraint, Predicate condition) + throws InterruptedException, ManagedLedgerException { + return null; + } + + @Override + public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + AsyncCallbacks.FindEntryCallback callback, Object ctx) { + + } + + @Override + public void resetCursor(Position position) throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void asyncResetCursor(Position position, boolean forceReset, AsyncCallbacks.ResetCursorCallback callback) { + + } + + @Override + public List replayEntries(Set positions) + throws InterruptedException, ManagedLedgerException { + return null; + } + + @Override + public Set asyncReplayEntries(Set positions, + AsyncCallbacks.ReadEntriesCallback callback, Object ctx) { + return null; + } + + @Override + public Set asyncReplayEntries(Set positions, + AsyncCallbacks.ReadEntriesCallback callback, Object ctx, + boolean sortEntries) { + return null; + } + + @Override + public void close() throws InterruptedException, ManagedLedgerException { + + } + + @Override + public void asyncClose(AsyncCallbacks.CloseCallback callback, Object ctx) { + + } + + @Override + public Position getFirstPosition() { + return null; + } + + @Override + public void setActive() { + + } + + @Override + public void setInactive() { + + } + + @Override + public void setAlwaysInactive() { + + } + + @Override + public boolean isActive() { + return false; + } + + @Override + public boolean isDurable() { + return false; + } + + @Override + public long getNumberOfEntriesSinceFirstNotAckedMessage() { + return 0; + } + + @Override + public int getTotalNonContiguousDeletedMessagesRange() { + return 0; + } + + @Override + public int getNonContiguousDeletedMessagesRangeSerializedSize() { + return 0; + } + + @Override + public long getEstimatedSizeSinceMarkDeletePosition() { + return 0; + } + + @Override + public double getThrottleMarkDelete() { + return 0; + } + + @Override + public void setThrottleMarkDelete(double throttleMarkDelete) { + + } + + @Override + public ManagedLedger getManagedLedger() { + return null; + } + + @Override + public Range getLastIndividualDeletedRange() { + return null; + } + + @Override + public void trimDeletedEntries(List entries) { + + } + + @Override + public long[] getDeletedBatchIndexesAsLongArray(PositionImpl position) { + return new long[0]; + } + + @Override + public ManagedCursorMXBean getStats() { + return null; + } + + @Override + public boolean checkAndUpdateReadPositionChanged() { + return false; + } + + @Override + public boolean isClosed() { + return false; + } +} From 7de411f17c76238e1fd620165b2ef8ce98db39a9 Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 20 Sep 2022 17:31:22 +0800 Subject: [PATCH 02/18] fix test code --- .../broker/delayed/BucketDelayedDeliveryTracker.java | 2 +- .../pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java | 1 - .../apache/pulsar/broker/delayed/MockManagedCursor.java | 7 ++++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 0a04a29d5a4fc..8fd26938c26b7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -61,7 +61,7 @@ @ThreadSafe public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { - public static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar_internal.delayed.bucket"; + public static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket"; public static final String DELIMITER = "_"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 8c8e6271f2797..22155ff2e59a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -310,7 +310,6 @@ public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict(DelayedDe // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has // passed where it would have been triggered if the tick time was doing the triggering. Thread.sleep(1000); - verify(dispatcher).getCursor(); // Not wait for the message delivery to get triggered. Awaitility.await().atMost(10, TimeUnit.SECONDS) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index 51b0aee748f9b..06a33410ba36e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -38,7 +38,7 @@ public class MockManagedCursor implements ManagedCursor { private final String name; - private Map cursorProperties; + private final Map cursorProperties; public MockManagedCursor(String name) { this.name = name; @@ -76,6 +76,11 @@ public CompletableFuture putCursorProperty(String key, String value) { return CompletableFuture.completedFuture(null); } + @Override + public CompletableFuture setCursorProperties(Map cursorProperties) { + return CompletableFuture.completedFuture(null); + } + public CompletableFuture removeCursorProperty(String key) { cursorProperties.remove(key); return CompletableFuture.completedFuture(null); From 91edc140e58f9b000e995e92c22a20331d8db967 Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 10 Oct 2022 20:01:00 +0800 Subject: [PATCH 03/18] rebase master & remove recover --- .../delayed/BucketDelayedDeliveryTracker.java | 93 +++---------------- .../InMemoryDelayedDeliveryTracker.java | 4 + .../BuketDelayedDeliveryTrackerTest.java | 67 ++++--------- .../delayed/InMemoryDeliveryTrackerTest.java | 14 +-- .../broker/delayed/MockManagedCursor.java | 9 +- 5 files changed, 49 insertions(+), 138 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 8fd26938c26b7..2a79b00a45df7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -48,13 +48,11 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.mutable.MutableLong; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @Slf4j @@ -88,20 +86,24 @@ public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, + long fixedDelayDetectionLookahead, BucketSnapshotStorage bucketSnapshotStorage, long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, int maxNumBuckets) { this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, + fixedDelayDetectionLookahead, bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegment, maxNumBuckets); } BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict, + long fixedDelayDetectionLookahead, BucketSnapshotStorage bucketSnapshotStorage, long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, int maxNumBuckets) { - super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); + super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict, + fixedDelayDetectionLookahead); this.minIndexCountPerBucket = minIndexCountPerBucket; this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; this.maxNumBuckets = maxNumBuckets; @@ -112,40 +114,11 @@ public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker this.bucketSnapshotStorage = bucketSnapshotStorage; - numberDelayedMessages = recoverBucketSnapshot(); + numberDelayedMessages = 0L; this.lastMutableBucket = new Bucket(-1L, -1L, new HashMap<>()); } - @SneakyThrows - private long recoverBucketSnapshot() { - List> completableFutures = new ArrayList<>(); - this.cursor.getCursorProperties().keySet().forEach(key -> { - if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) { - String[] keys = key.split(DELIMITER); - checkArgument(keys.length == 3); - Bucket bucket = createImmutableBucket(Long.parseLong(keys[1]), Long.parseLong(keys[2])); - completableFutures.add(asyncLoadNextBucketSnapshotEntry(bucket, true)); - } - }); - - if (completableFutures.isEmpty()) { - return 0; - } - - FutureUtil.waitForAll(completableFutures).get(); - - MutableLong numberDelayedMessages = new MutableLong(0); - immutableBuckets.asMapOfRanges().values().forEach(bucket -> { - numberDelayedMessages.add(bucket.numberBucketDelayedMessages); - }); - - log.info("[{}] Recover delayed message index bucket snapshot finish, buckets: {}, numberDelayedMessages: {}", - dispatcher.getName(), immutableBuckets.asMapOfRanges().size(), numberDelayedMessages.getValue()); - - return numberDelayedMessages.getValue(); - } - private void moveScheduledMessageToSharedQueue(long cutoffTime) { TripleLongPriorityQueue priorityQueue = getPriorityQueue(); while (!priorityQueue.isEmpty()) { @@ -318,7 +291,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { @SneakyThrows - private CompletableFuture asyncLoadNextBucketSnapshotEntry(Bucket bucket, boolean isRebuild) { + private CompletableFuture asyncLoadNextBucketSnapshotEntry(Bucket bucket, boolean isRecover) { if (log.isDebugEnabled()) { log.debug("[{}] Load next bucket snapshot data, bucket: {}", dispatcher.getName(), bucket); } @@ -337,28 +310,8 @@ private CompletableFuture asyncLoadNextBucketSnapshotEntry(Bucket bucket, Objects.requireNonNull(bucketId); CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); - if (isRebuild) { - final long cutoffTime = getCutoffTime(); - // Load Metadata of bucket snapshot - bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId).thenAccept(snapshotMetadata -> { - List metadataList = snapshotMetadata.getMetadataListList(); - - // Skip all already reach schedule time snapshot segments - int nextSnapshotEntryIndex = 0; - while (nextSnapshotEntryIndex < metadataList.size() - && metadataList.get(nextSnapshotEntryIndex).getMaxScheduleTimestamp() <= cutoffTime) { - nextSnapshotEntryIndex++; - } - - final int lastSegmentEntryId = metadataList.size(); - - long numberMessages = bucket.covertDelayIndexMapAndCount(nextSnapshotEntryIndex, metadataList); - bucket.setNumberBucketDelayedMessages(numberMessages); - bucket.setLastSegmentEntryId(lastSegmentEntryId); - - int nextSegmentEntryId = nextSnapshotEntryIndex + 1; - loadMetaDataFuture.complete(nextSegmentEntryId); - }); + if (isRecover) { + // TODO Recover bucket snapshot } else { loadMetaDataFuture.complete(bucket.currentSegmentEntryId + 1); } @@ -379,28 +332,12 @@ private CompletableFuture asyncLoadNextBucketSnapshotEntry(Bucket bucket, List indexList = snapshotSegment.getIndexesList(); DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); - // Rebuild delayed message index bucket load data in parallel, so should be use synchronized - // to ensure data consistency - if (isRebuild) { - synchronized (snapshotSegmentLastIndexTable) { - this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), - lastDelayedIndex.getEntryId(), bucket); - } - - synchronized (sharedBucketPriorityQueue) { - for (DelayedIndex index : indexList) { - sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), - index.getEntryId()); - } - } - } else { - this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), - lastDelayedIndex.getEntryId(), bucket); - - for (DelayedIndex index : indexList) { - sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), - index.getEntryId()); - } + this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), + lastDelayedIndex.getEntryId(), bucket); + + for (DelayedIndex index : indexList) { + sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), + index.getEntryId()); } bucket.setCurrentSegmentEntryId(nextSegmentEntryId); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index ef8fc2abf2bec..40a6d8e8b61b5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.delayed; +import com.google.common.annotations.VisibleForTesting; import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; @@ -25,6 +26,7 @@ import java.util.NavigableSet; import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; @@ -59,6 +61,8 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T // always going to be in FIFO order, then we can avoid pulling all the messages in // tracker. Instead, we use the lookahead for detection and pause the read from // the cursor if the delays are fixed. + @Getter + @VisibleForTesting private final long fixedDelayDetectionLookahead; // This is the timestamp of the message with the highest delivery time diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java index 84273b69b4231..9525ba6b72aa8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; @@ -76,7 +75,7 @@ public Object[][] provider(Method method) throws Exception { return switch (methodName) { case "test" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + false, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testWithTimer" -> { Timer timer = mock(Timer.class); @@ -104,33 +103,39 @@ public Object[][] provider(Method method) throws Exception { yield new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50), + false, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50), tasks }}; } case "testAddWithinTickTime" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock, - false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + false, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testAddMessageWithStrictDelay" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock, - true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, - true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; - case "testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict", "testRecoverSnapshot" -> new Object[][]{{ - new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, - true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) - }}; - case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict", "testExistDelayedMessage" -> new Object[][]{{ + case "testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict", "testRecoverSnapshot" -> + new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, + true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict", "testExistDelayedMessage" -> + new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, + true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + }}; + case "testWithFixedDelays", "testWithMixedDelays", "testWithNoDelays" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, - true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + true, 100, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; default -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), 50) + true, 0, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), 50) }}; }; } @@ -158,40 +163,4 @@ public void testContainsMessage(DelayedDeliveryTracker tracker) { tracker.close(); } - - @Test(dataProvider = "delayedTracker") - public void testRecoverSnapshot(DelayedDeliveryTracker tracker) throws Exception { - for (int i = 1; i <= 100; i++) { - tracker.addMessage(i, i, i * 10); - } - - assertEquals(tracker.getNumberOfDelayedMessages(), 100); - - clockTime.set(1 * 10); - - assertTrue(tracker.hasMessageAvailable()); - Set scheduledMessages = tracker.getScheduledMessages(100); - - assertEquals(scheduledMessages.size(), 1); - - tracker.addMessage(101, 101, 101 * 10); - - tracker.close(); - - clockTime.set(30 * 10); - - tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, - true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50); - - assertFalse(tracker.containsMessage(101, 101)); - assertEquals(tracker.getNumberOfDelayedMessages(), 70); - - clockTime.set(100 * 10); - - assertTrue(tracker.hasMessageAvailable()); - scheduledMessages = tracker.getScheduledMessages(70); - - assertEquals(scheduledMessages.size(), 70); - tracker.close(); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 22155ff2e59a7..7d50c2a05cdc6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -319,7 +319,7 @@ public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict(DelayedDe } @Test(dataProvider = "delayedTracker") - public void testWithFixedDelays(DelayedDeliveryTracker tracker) throws Exception { + public void testWithFixedDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); assertTrue(tracker.addMessage(1, 1, 10)); @@ -332,13 +332,13 @@ public void testWithFixedDelays(DelayedDeliveryTracker tracker) throws Exception assertEquals(tracker.getNumberOfDelayedMessages(), 5); assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= fixedDelayLookahead; i++) { + for (int i = 6; i <= tracker.getFixedDelayDetectionLookahead(); i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } assertTrue(tracker.shouldPauseAllDeliveries()); - clockTime.set(fixedDelayLookahead * 10); + clockTime.set(tracker.getFixedDelayDetectionLookahead() * 10); tracker.getScheduledMessages(100); @@ -356,7 +356,7 @@ public void testWithFixedDelays(DelayedDeliveryTracker tracker) throws Exception } @Test(dataProvider = "delayedTracker") - public void testWithMixedDelays(DelayedDeliveryTracker tracker) throws Exception { + public void testWithMixedDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); assertTrue(tracker.addMessage(1, 1, 10)); @@ -367,7 +367,7 @@ public void testWithMixedDelays(DelayedDeliveryTracker tracker) throws Exception assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= fixedDelayLookahead; i++) { + for (int i = 6; i <= tracker.getFixedDelayDetectionLookahead(); i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } @@ -382,7 +382,7 @@ public void testWithMixedDelays(DelayedDeliveryTracker tracker) throws Exception } @Test(dataProvider = "delayedTracker") - public void testWithNoDelays(DelayedDeliveryTracker tracker) throws Exception { + public void testWithNoDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); assertTrue(tracker.addMessage(1, 1, 10)); @@ -393,7 +393,7 @@ public void testWithNoDelays(DelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= fixedDelayLookahead; i++) { + for (int i = 6; i <= tracker.getFixedDelayDetectionLookahead(); i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index 06a33410ba36e..945eb183ec7e4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.delayed; -import com.google.common.base.Predicate; import com.google.common.collect.Range; import java.util.List; import java.util.Map; @@ -259,18 +258,20 @@ public void asyncSkipEntries(int numEntriesToSkip, IndividualDeletedEntries dele } @Override - public Position findNewestMatching(Predicate condition) throws InterruptedException, ManagedLedgerException { + public Position findNewestMatching(java.util.function.Predicate condition) + throws InterruptedException, ManagedLedgerException { return null; } @Override - public Position findNewestMatching(FindPositionConstraint constraint, Predicate condition) + public Position findNewestMatching(FindPositionConstraint constraint, java.util.function.Predicate condition) throws InterruptedException, ManagedLedgerException { return null; } @Override - public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + public void asyncFindNewestMatching(FindPositionConstraint constraint, + java.util.function.Predicate condition, AsyncCallbacks.FindEntryCallback callback, Object ctx) { } From c9446243dafb860c2437ae7079fe050ccdde30b5 Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 10 Oct 2022 23:18:12 +0800 Subject: [PATCH 04/18] Address comment --- .../delayed/BucketDelayedDeliveryTracker.java | 225 +++++++++--------- .../delayed/{Bucket.java => BucketState.java} | 45 +--- .../InMemoryDelayedDeliveryTracker.java | 2 +- 3 files changed, 123 insertions(+), 149 deletions(-) rename pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/{Bucket.java => BucketState.java} (60%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 2a79b00a45df7..d67740d8b8024 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -40,8 +40,10 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import javax.annotation.concurrent.ThreadSafe; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; @@ -59,6 +61,8 @@ @ThreadSafe public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { + protected static final int AsyncOperationTimeoutSeconds = 30; + public static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket"; public static final String DELIMITER = "_"; @@ -75,13 +79,13 @@ public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker private long numberDelayedMessages; - private final Bucket lastMutableBucket; + private final BucketState lastMutableBucketState; private final TripleLongPriorityQueue sharedBucketPriorityQueue; - private final RangeMap immutableBuckets; + private final RangeMap immutableBuckets; - private final Table snapshotSegmentLastIndexTable; + private final Table snapshotSegmentLastIndexTable; BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, @@ -114,9 +118,9 @@ public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker this.bucketSnapshotStorage = bucketSnapshotStorage; - numberDelayedMessages = 0L; + this.numberDelayedMessages = 0L; - this.lastMutableBucket = new Bucket(-1L, -1L, new HashMap<>()); + this.lastMutableBucketState = new BucketState(-1L, -1L); } private void moveScheduledMessageToSharedQueue(long cutoffTime) { @@ -143,7 +147,7 @@ public void run(Timeout timeout) throws Exception { super.run(timeout); } - private Optional findBucket(long ledgerId) { + private Optional findBucket(long ledgerId) { if (immutableBuckets.asMapOfRanges().isEmpty()) { return Optional.empty(); } @@ -158,25 +162,26 @@ private Optional findBucket(long ledgerId) { private Long getBucketIdByBucketKey(String bucketKey) { String bucketIdStr = cursor.getCursorProperties().get(bucketKey); if (StringUtils.isBlank(bucketIdStr)) { - return null; + return -1L; } return Long.valueOf(bucketIdStr); } - private Bucket createImmutableBucket(long startLedgerId, long endLedgerId) { - Bucket newBucket = new Bucket(startLedgerId, endLedgerId, new HashMap<>()); - immutableBuckets.put(Range.closed(startLedgerId, endLedgerId), newBucket); - return newBucket; + private BucketState createImmutableBucket(long startLedgerId, long endLedgerId) { + BucketState bucketState = new BucketState(startLedgerId, endLedgerId); + immutableBuckets.put(Range.closed(startLedgerId, endLedgerId), bucketState); + return bucketState; } private CompletableFuture asyncSaveBucketSnapshot( - final String bucketKey, SnapshotMetadata snapshotMetadata, + BucketState bucketState, SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments) { - Long bucketId = getBucketIdByBucketKey(bucketKey); - checkArgument(bucketId == null); return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments) - .thenCompose(newBucketId -> putBucketKeyId(bucketKey, newBucketId)); + .thenCompose(newBucketId -> { + bucketState.setBucketId(newBucketId); + return putBucketKeyId(bucketState.bucketKey(), newBucketId); + }); } private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { @@ -188,12 +193,12 @@ private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) private CompletableFuture asyncCreateBucketSnapshot() { TripleLongPriorityQueue priorityQueue = super.getPriorityQueue(); if (priorityQueue.isEmpty()) { - return CompletableFuture.completedFuture(null); + return CompletableFuture.completedFuture(-1L); } long numMessages = 0; - final long startLedgerId = lastMutableBucket.startLedgerId; - final long endLedgerId = lastMutableBucket.endLedgerId; + final long startLedgerId = lastMutableBucketState.startLedgerId; + final long endLedgerId = lastMutableBucketState.endLedgerId; List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); @@ -213,7 +218,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); - // Move first segment of bucket snapshot to sharedBucketPriorityQueue + // Move first segment of bucketState snapshot to sharedBucketPriorityQueue if (segmentMetadataList.size() == 0) { sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); } @@ -226,9 +231,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { .setLedgerId(ledgerId) .setEntryId(entryId).build(); - if (entryId <= Integer.MAX_VALUE) { - bitMap.compute(ledgerId, (k, v) -> new BitSet()).set((int) entryId); - } + bitMap.computeIfAbsent(ledgerId, k -> new BitSet()).set((int) entryId); snapshotSegmentBuilder.addIndexes(delayedIndex); @@ -258,28 +261,27 @@ private CompletableFuture asyncCreateBucketSnapshot() { final int lastSegmentEntryId = segmentMetadataList.size(); - Bucket bucket = this.createImmutableBucket(startLedgerId, endLedgerId); - bucket.setCurrentSegmentEntryId(1); - bucket.setNumberBucketDelayedMessages(numMessages); - bucket.setLastSegmentEntryId(lastSegmentEntryId); + BucketState bucketState = this.createImmutableBucket(startLedgerId, endLedgerId); + bucketState.setCurrentSegmentEntryId(1); + bucketState.setNumberBucketDelayedMessages(numMessages); + bucketState.setLastSegmentEntryId(lastSegmentEntryId); // Add the first snapshot segment last message to snapshotSegmentLastMessageTable checkArgument(!bucketSnapshotSegments.isEmpty()); SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); DelayedIndex delayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); - snapshotSegmentLastIndexTable.put(delayedIndex.getLedgerId(), delayedIndex.getEntryId(), bucket); + snapshotSegmentLastIndexTable.put(delayedIndex.getLedgerId(), delayedIndex.getEntryId(), bucketState); if (log.isDebugEnabled()) { - log.debug("[{}] Create bucket snapshot, bucket: {}", dispatcher.getName(), bucket); + log.debug("[{}] Create bucketState snapshot, bucketState: {}", dispatcher.getName(), bucketState); } - String bucketKey = bucket.bucketKey(); - CompletableFuture future = asyncSaveBucketSnapshot(bucketKey, + CompletableFuture future = asyncSaveBucketSnapshot(bucketState, bucketSnapshotMetadata, bucketSnapshotSegments); - bucket.setSnapshotCreateFuture(future); + bucketState.setSnapshotCreateFuture(future); future.whenComplete((__, ex) -> { if (ex == null) { - bucket.setSnapshotCreateFuture(null); + bucketState.setSnapshotCreateFuture(null); } else { //TODO Record create snapshot failed log.error("Failed to create snapshot: ", ex); @@ -290,65 +292,72 @@ private CompletableFuture asyncCreateBucketSnapshot() { } - @SneakyThrows - private CompletableFuture asyncLoadNextBucketSnapshotEntry(Bucket bucket, boolean isRecover) { + private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState bucketState, boolean isRecover) { if (log.isDebugEnabled()) { - log.debug("[{}] Load next bucket snapshot data, bucket: {}", dispatcher.getName(), bucket); + log.debug("[{}] Load next bucketState snapshot data, bucketState: {}", dispatcher.getName(), bucketState); } - if (bucket == null) { + if (bucketState == null) { return CompletableFuture.completedFuture(null); } - final CompletableFuture createFuture = bucket.snapshotCreateFuture; - if (createFuture != null) { - // Wait bucket snapshot create finish - createFuture.get(); - } - - final String bucketKey = bucket.bucketKey(); - final Long bucketId = getBucketIdByBucketKey(bucketKey); - Objects.requireNonNull(bucketId); - - CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); - if (isRecover) { - // TODO Recover bucket snapshot + CompletableFuture createFuture; + if (bucketState.snapshotCreateFuture != null) { + // Wait bucketState snapshot create finish + createFuture = bucketState.snapshotCreateFuture; } else { - loadMetaDataFuture.complete(bucket.currentSegmentEntryId + 1); + createFuture = CompletableFuture.completedFuture(null); } - CompletableFuture future = loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { - if (nextSegmentEntryId > bucket.lastSegmentEntryId) { - // TODO Delete bucket snapshot - return CompletableFuture.completedFuture(null); + return createFuture.thenCompose(__ -> { + final String bucketKey = bucketState.bucketKey(); + final Long bucketId; + if (bucketState.getBucketId() != -1L) { + bucketId = bucketState.getBucketId(); + } else { + bucketId = getBucketIdByBucketKey(bucketKey); + bucketState.setBucketId(bucketId); } - return bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId) - .thenAccept(bucketSnapshotSegments -> { - if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { - return; - } + CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); + if (isRecover) { + // TODO Recover bucketState snapshot + } else { + loadMetaDataFuture.complete(bucketState.currentSegmentEntryId + 1); + } - SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - List indexList = snapshotSegment.getIndexesList(); - DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); + return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { + if (nextSegmentEntryId > bucketState.lastSegmentEntryId) { + // TODO Delete bucketState snapshot + return CompletableFuture.completedFuture(null); + } - this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), - lastDelayedIndex.getEntryId(), bucket); + return bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId) + .thenAccept(bucketSnapshotSegments -> { + if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { + return; + } - for (DelayedIndex index : indexList) { - sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), - index.getEntryId()); - } + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); + List indexList = snapshotSegment.getIndexesList(); + DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); + + this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), + lastDelayedIndex.getEntryId(), bucketState); + + for (DelayedIndex index : indexList) { + sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), + index.getEntryId()); + } - bucket.setCurrentSegmentEntryId(nextSegmentEntryId); - }); + bucketState.setCurrentSegmentEntryId(nextSegmentEntryId); + }); + }); }); - return future; } private void resetLastMutableBucketRange() { - lastMutableBucket.setStartLedgerId(-1L); - lastMutableBucket.setEndLedgerId(-1L); + lastMutableBucketState.setStartLedgerId(-1L); + lastMutableBucketState.setEndLedgerId(-1L); } @Override @@ -366,7 +375,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver boolean existBucket = findBucket(ledgerId).isPresent(); // Create bucket snapshot - if (ledgerId > lastMutableBucket.endLedgerId && !getPriorityQueue().isEmpty()) { + if (ledgerId > lastMutableBucketState.endLedgerId && !getPriorityQueue().isEmpty()) { if (getPriorityQueue().size() >= minIndexCountPerBucket || existBucket) { if (immutableBuckets.asMapOfRanges().size() >= maxNumBuckets) { // TODO merge bucket snapshot (synchronize operate) @@ -377,23 +386,23 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver } } - if (ledgerId < lastMutableBucket.startLedgerId || existBucket) { + if (ledgerId < lastMutableBucketState.startLedgerId || existBucket) { // If (ledgerId < startLedgerId || existBucket) means that message index belong to previous bucket range, // enter sharedBucketPriorityQueue directly sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); } else { - checkArgument(ledgerId >= lastMutableBucket.endLedgerId); + checkArgument(ledgerId >= lastMutableBucketState.endLedgerId); getPriorityQueue().add(deliverAt, ledgerId, entryId); - if (lastMutableBucket.startLedgerId == -1L) { - lastMutableBucket.setStartLedgerId(ledgerId); + if (lastMutableBucketState.startLedgerId == -1L) { + lastMutableBucketState.setStartLedgerId(ledgerId); } - lastMutableBucket.setEndLedgerId(ledgerId); + lastMutableBucketState.setEndLedgerId(ledgerId); } // TODO If the bitSet is sparse, this memory cost very high to deduplication and skip read message - lastMutableBucket.putIndexBit(ledgerId, entryId); + lastMutableBucketState.putIndexBit(ledgerId, entryId); numberDelayedMessages++; if (log.isDebugEnabled()) { @@ -445,7 +454,6 @@ public synchronized long getBufferMemoryUsage() { } @Override - @SneakyThrows public synchronized Set getScheduledMessages(int maxMessages) { long cutoffTime = getCutoffTime(); @@ -467,13 +475,18 @@ public synchronized Set getScheduledMessages(int maxMessages) { sharedBucketPriorityQueue.pop(); removeIndexBit(ledgerId, entryId); - Bucket bucket = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); - if (bucket != null && bucket.active) { + BucketState bucketState = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); + if (bucketState != null && bucketState.active) { if (log.isDebugEnabled()) { - log.debug("[{}] Load next snapshot segment, bucket: {}", dispatcher.getName(), bucket); + log.debug("[{}] Load next snapshot segment, bucketState: {}", dispatcher.getName(), bucketState); } // All message of current snapshot segment are scheduled, load next snapshot segment - asyncLoadNextBucketSnapshotEntry(bucket, false).get(); + try { + asyncLoadNextBucketSnapshotEntry(bucketState, false).get(AsyncOperationTimeoutSeconds, + TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } } --n; @@ -492,44 +505,42 @@ public synchronized Set getScheduledMessages(int maxMessages) { } @Override - @SneakyThrows public synchronized void clear() { super.clear(); cleanImmutableBuckets(true); sharedBucketPriorityQueue.clear(); resetLastMutableBucketRange(); - lastMutableBucket.delayedIndexBitMap.clear(); + lastMutableBucketState.delayedIndexBitMap.clear(); snapshotSegmentLastIndexTable.clear(); numberDelayedMessages = 0; } @Override - @SneakyThrows public synchronized void close() { super.close(); cleanImmutableBuckets(false); - lastMutableBucket.delayedIndexBitMap.clear(); + lastMutableBucketState.delayedIndexBitMap.clear(); sharedBucketPriorityQueue.close(); } private void cleanImmutableBuckets(boolean delete) { if (immutableBuckets != null) { - Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); + Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); while (iterator.hasNext()) { - Bucket bucket = iterator.next(); - if (bucket.delayedIndexBitMap != null) { - bucket.delayedIndexBitMap.clear(); + BucketState bucketState = iterator.next(); + if (bucketState.delayedIndexBitMap != null) { + bucketState.delayedIndexBitMap.clear(); } - CompletableFuture snapshotGenerateFuture = bucket.snapshotCreateFuture; + CompletableFuture snapshotGenerateFuture = bucketState.snapshotCreateFuture; if (snapshotGenerateFuture != null) { if (delete) { snapshotGenerateFuture.cancel(true); - // TODO delete bucket snapshot + // TODO delete bucketState snapshot } else { try { - snapshotGenerateFuture.get(); + snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); } catch (Exception e) { - log.warn("Failed wait to snapshot generate, bucket: {}", bucket); + log.warn("Failed wait to snapshot generate, bucketState: {}", bucketState); } } } @@ -539,27 +550,21 @@ private void cleanImmutableBuckets(boolean delete) { } private boolean removeIndexBit(long ledgerId, long entryId) { - if (entryId > Integer.MAX_VALUE) { - return false; - } - - if (lastMutableBucket.removeIndexBit(ledgerId, (int) entryId)) { + if (lastMutableBucketState.removeIndexBit(ledgerId, (int) entryId)) { return true; } - return findBucket(ledgerId).map(bucket -> bucket.removeIndexBit(ledgerId, (int) entryId)).orElse(false); + return findBucket(ledgerId).map(bucketState -> bucketState.removeIndexBit(ledgerId, (int) entryId)) + .orElse(false); } @Override public boolean containsMessage(long ledgerId, long entryId) { - if (entryId > Integer.MAX_VALUE) { - return false; - } - - if (lastMutableBucket.containsMessage(ledgerId, (int) entryId)) { + if (lastMutableBucketState.containsMessage(ledgerId, (int) entryId)) { return true; } - return findBucket(ledgerId).map(bucket -> bucket.containsMessage(ledgerId, (int) entryId)).orElse(false); + return findBucket(ledgerId).map(bucketState -> bucketState.containsMessage(ledgerId, (int) entryId)) + .orElse(false); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java similarity index 60% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/Bucket.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java index 4a3933f618e84..eb92ef694223a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java @@ -20,19 +20,16 @@ import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELIMITER; -import com.google.protobuf.ByteString; import java.util.BitSet; -import java.util.List; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; import lombok.AllArgsConstructor; import lombok.Data; -import org.apache.commons.lang3.mutable.MutableLong; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; @Data @AllArgsConstructor -public class Bucket { +public class BucketState { long startLedgerId; long endLedgerId; @@ -47,39 +44,17 @@ public class Bucket { long snapshotLength; + long bucketId; + boolean active; volatile CompletableFuture snapshotCreateFuture; - Bucket(long startLedgerId, long endLedgerId, Map delayedIndexBitMap) { - this(startLedgerId, endLedgerId, delayedIndexBitMap, -1, -1, 0, 0, true, null); - } - - long covertDelayIndexMapAndCount(int startSnapshotIndex, List segmentMetadata) { - delayedIndexBitMap.clear(); - MutableLong numberMessages = new MutableLong(0); - for (int i = startSnapshotIndex; i < segmentMetadata.size(); i++) { - Map bitByteStringMap = segmentMetadata.get(i).getDelayedIndexBitMapMap(); - bitByteStringMap.forEach((k, v) -> { - boolean exist = delayedIndexBitMap.containsKey(k); - byte[] bytes = v.toByteArray(); - BitSet bitSet = BitSet.valueOf(bytes); - numberMessages.add(bitSet.cardinality()); - if (!exist) { - delayedIndexBitMap.put(k, bitSet); - } else { - delayedIndexBitMap.get(k).or(bitSet); - } - }); - } - return numberMessages.longValue(); + BucketState(long startLedgerId, long endLedgerId) { + this(startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, -1, true, null); } boolean containsMessage(long ledgerId, int entryId) { - if (delayedIndexBitMap == null) { - return false; - } - BitSet bitSet = delayedIndexBitMap.get(ledgerId); if (bitSet == null) { return false; @@ -88,16 +63,10 @@ boolean containsMessage(long ledgerId, int entryId) { } void putIndexBit(long ledgerId, long entryId) { - if (entryId < Integer.MAX_VALUE) { - delayedIndexBitMap.compute(ledgerId, (k, v) -> new BitSet()).set((int) entryId); - } + delayedIndexBitMap.computeIfAbsent(ledgerId, k -> new BitSet()).set((int) entryId); } boolean removeIndexBit(long ledgerId, int entryId) { - if (delayedIndexBitMap == null) { - return false; - } - boolean contained = false; BitSet bitSet = delayedIndexBitMap.get(ledgerId); if (bitSet != null && bitSet.get(entryId)) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 40a6d8e8b61b5..c7db10157f1d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -288,11 +288,11 @@ public void run(Timeout timeout) throws Exception { @Override public void close() { + priorityQueue.close(); if (timeout != null) { timeout.cancel(); timeout = null; } - priorityQueue.close(); } @Override From 9f46790059a66c5e1db893a7eee3eb76fd8b70c8 Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 11 Oct 2022 23:02:44 +0800 Subject: [PATCH 05/18] remove active flag --- .../pulsar/broker/delayed/BucketDelayedDeliveryTracker.java | 2 +- .../java/org/apache/pulsar/broker/delayed/BucketState.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index d67740d8b8024..d15f1e0521dcf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -476,7 +476,7 @@ public synchronized Set getScheduledMessages(int maxMessages) { removeIndexBit(ledgerId, entryId); BucketState bucketState = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); - if (bucketState != null && bucketState.active) { + if (bucketState != null) { if (log.isDebugEnabled()) { log.debug("[{}] Load next snapshot segment, bucketState: {}", dispatcher.getName(), bucketState); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java index eb92ef694223a..c3c2310e2fb81 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java @@ -46,12 +46,10 @@ public class BucketState { long bucketId; - boolean active; - volatile CompletableFuture snapshotCreateFuture; BucketState(long startLedgerId, long endLedgerId) { - this(startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, -1, true, null); + this(startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, -1, null); } boolean containsMessage(long ledgerId, int entryId) { From ffb29bbdd373799d62611b2ea20a23f5a982c886 Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 11 Oct 2022 23:40:41 +0800 Subject: [PATCH 06/18] fix annotation --- .../delayed/BucketDelayedDeliveryTracker.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index d15f1e0521dcf..a2ca41cdddd85 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -218,7 +218,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); - // Move first segment of bucketState snapshot to sharedBucketPriorityQueue + // Move first segment of bucket snapshot to sharedBucketPriorityQueue if (segmentMetadataList.size() == 0) { sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); } @@ -273,7 +273,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { snapshotSegmentLastIndexTable.put(delayedIndex.getLedgerId(), delayedIndex.getEntryId(), bucketState); if (log.isDebugEnabled()) { - log.debug("[{}] Create bucketState snapshot, bucketState: {}", dispatcher.getName(), bucketState); + log.debug("[{}] Create bucket snapshot, bucketState: {}", dispatcher.getName(), bucketState); } CompletableFuture future = asyncSaveBucketSnapshot(bucketState, @@ -294,7 +294,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState bucketState, boolean isRecover) { if (log.isDebugEnabled()) { - log.debug("[{}] Load next bucketState snapshot data, bucketState: {}", dispatcher.getName(), bucketState); + log.debug("[{}] Load next bucket snapshot data, bucketState: {}", dispatcher.getName(), bucketState); } if (bucketState == null) { return CompletableFuture.completedFuture(null); @@ -302,7 +302,7 @@ private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState buc CompletableFuture createFuture; if (bucketState.snapshotCreateFuture != null) { - // Wait bucketState snapshot create finish + // Wait bucket snapshot create finish createFuture = bucketState.snapshotCreateFuture; } else { createFuture = CompletableFuture.completedFuture(null); @@ -320,14 +320,14 @@ private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState buc CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); if (isRecover) { - // TODO Recover bucketState snapshot + // TODO Recover bucket snapshot } else { loadMetaDataFuture.complete(bucketState.currentSegmentEntryId + 1); } return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { if (nextSegmentEntryId > bucketState.lastSegmentEntryId) { - // TODO Delete bucketState snapshot + // TODO Delete bucket snapshot return CompletableFuture.completedFuture(null); } @@ -535,7 +535,7 @@ private void cleanImmutableBuckets(boolean delete) { if (snapshotGenerateFuture != null) { if (delete) { snapshotGenerateFuture.cancel(true); - // TODO delete bucketState snapshot + // TODO delete bucket snapshot } else { try { snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); From 51aabc65d3ea6411d44be4104f6e4ab62403f2ec Mon Sep 17 00:00:00 2001 From: coderzc Date: Wed, 12 Oct 2022 11:28:28 +0800 Subject: [PATCH 07/18] improve code --- .../delayed/BucketDelayedDeliveryTracker.java | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index a2ca41cdddd85..8407ead083cea 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -159,12 +159,20 @@ private Optional findBucket(long ledgerId) { return Optional.ofNullable(immutableBuckets.get(ledgerId)); } - private Long getBucketIdByBucketKey(String bucketKey) { - String bucketIdStr = cursor.getCursorProperties().get(bucketKey); + private Long getBucketId(BucketState bucketState) { + long bucketId = bucketState.getBucketId(); + if (bucketId != -1L) { + return bucketId; + } + + String bucketIdStr = cursor.getCursorProperties().get(bucketState.bucketKey()); if (StringUtils.isBlank(bucketIdStr)) { return -1L; } - return Long.valueOf(bucketIdStr); + + bucketId = Long.parseLong(bucketIdStr); + bucketState.setBucketId(bucketId); + return bucketId; } private BucketState createImmutableBucket(long startLedgerId, long endLedgerId) { @@ -178,16 +186,21 @@ private CompletableFuture asyncSaveBucketSnapshot( List bucketSnapshotSegments) { return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments) - .thenCompose(newBucketId -> { + .thenApply(newBucketId -> { bucketState.setBucketId(newBucketId); - return putBucketKeyId(bucketState.bucketKey(), newBucketId); + String bucketKey = bucketState.bucketKey(); + putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { + log.warn("Failed to record bucketId to cursor property, bucketKey: {}", bucketKey); + return null; + }); + return newBucketId; }); } - private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { + private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { Objects.requireNonNull(bucketId); return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), - ManagedLedgerException.BadVersionException.class).thenApply(__ -> bucketId); + ManagedLedgerException.BadVersionException.class); } private CompletableFuture asyncCreateBucketSnapshot() { @@ -300,24 +313,14 @@ private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState buc return CompletableFuture.completedFuture(null); } - CompletableFuture createFuture; - if (bucketState.snapshotCreateFuture != null) { - // Wait bucket snapshot create finish - createFuture = bucketState.snapshotCreateFuture; - } else { - createFuture = CompletableFuture.completedFuture(null); + // Wait bucket snapshot create finish + CompletableFuture snapshotCreateFuture = bucketState.snapshotCreateFuture; + if (snapshotCreateFuture == null) { + snapshotCreateFuture = CompletableFuture.completedFuture(-1L); } - return createFuture.thenCompose(__ -> { - final String bucketKey = bucketState.bucketKey(); - final Long bucketId; - if (bucketState.getBucketId() != -1L) { - bucketId = bucketState.getBucketId(); - } else { - bucketId = getBucketIdByBucketKey(bucketKey); - bucketState.setBucketId(bucketId); - } - + return snapshotCreateFuture.thenCompose(__ -> { + final Long bucketId = getBucketId(bucketState); CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); if (isRecover) { // TODO Recover bucket snapshot @@ -377,12 +380,11 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver // Create bucket snapshot if (ledgerId > lastMutableBucketState.endLedgerId && !getPriorityQueue().isEmpty()) { if (getPriorityQueue().size() >= minIndexCountPerBucket || existBucket) { - if (immutableBuckets.asMapOfRanges().size() >= maxNumBuckets) { - // TODO merge bucket snapshot (synchronize operate) - } - asyncCreateBucketSnapshot(); resetLastMutableBucketRange(); + if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { + // TODO merge bucket snapshot (synchronize operate) + } } } From bdd3dac36ddfebd67b40d7d0e41a1fc71fbd5964 Mon Sep 17 00:00:00 2001 From: coderzc Date: Wed, 12 Oct 2022 18:21:18 +0800 Subject: [PATCH 08/18] Use RoaringBitmap instead of BitSet --- .../delayed/BucketDelayedDeliveryTracker.java | 25 ++++++++++--------- .../pulsar/broker/delayed/BucketState.java | 20 +++++++-------- ...ayedMessageIndexBucketSnapshotFormat.proto | 4 +-- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 8407ead083cea..8003ebd23ceb2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -28,9 +28,9 @@ import com.google.protobuf.ByteString; import io.netty.util.Timeout; import io.netty.util.Timer; +import java.nio.ByteBuffer; import java.time.Clock; import java.util.ArrayList; -import java.util.BitSet; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -56,6 +56,7 @@ import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; +import org.roaringbitmap.RoaringBitmap; @Slf4j @ThreadSafe @@ -215,7 +216,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); - Map bitMap = new HashMap<>(); + Map bitMap = new HashMap<>(); SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); @@ -244,7 +245,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { .setLedgerId(ledgerId) .setEntryId(entryId).build(); - bitMap.computeIfAbsent(ledgerId, k -> new BitSet()).set((int) entryId); + bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); snapshotSegmentBuilder.addIndexes(delayedIndex); @@ -252,11 +253,12 @@ private CompletableFuture asyncCreateBucketSnapshot() { segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); currentTimestampUpperLimit = 0; - Iterator> iterator = bitMap.entrySet().iterator(); + Iterator> iterator = bitMap.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - ByteString byteString = ByteString.copyFrom(entry.getValue().toByteArray()); - segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), byteString); + Map.Entry entry = iterator.next(); + byte[] array = new byte[entry.getValue().serializedSizeInBytes()]; + entry.getValue().serialize(ByteBuffer.wrap(array)); + segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array)); iterator.remove(); } @@ -403,7 +405,6 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver lastMutableBucketState.setEndLedgerId(ledgerId); } - // TODO If the bitSet is sparse, this memory cost very high to deduplication and skip read message lastMutableBucketState.putIndexBit(ledgerId, entryId); numberDelayedMessages++; @@ -552,21 +553,21 @@ private void cleanImmutableBuckets(boolean delete) { } private boolean removeIndexBit(long ledgerId, long entryId) { - if (lastMutableBucketState.removeIndexBit(ledgerId, (int) entryId)) { + if (lastMutableBucketState.removeIndexBit(ledgerId, entryId)) { return true; } - return findBucket(ledgerId).map(bucketState -> bucketState.removeIndexBit(ledgerId, (int) entryId)) + return findBucket(ledgerId).map(bucketState -> bucketState.removeIndexBit(ledgerId, entryId)) .orElse(false); } @Override public boolean containsMessage(long ledgerId, long entryId) { - if (lastMutableBucketState.containsMessage(ledgerId, (int) entryId)) { + if (lastMutableBucketState.containsMessage(ledgerId, entryId)) { return true; } - return findBucket(ledgerId).map(bucketState -> bucketState.containsMessage(ledgerId, (int) entryId)) + return findBucket(ledgerId).map(bucketState -> bucketState.containsMessage(ledgerId, entryId)) .orElse(false); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java index c3c2310e2fb81..b42375cbb53ae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java @@ -20,12 +20,12 @@ import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELIMITER; -import java.util.BitSet; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; import lombok.AllArgsConstructor; import lombok.Data; +import org.roaringbitmap.RoaringBitmap; @Data @AllArgsConstructor @@ -34,7 +34,7 @@ public class BucketState { long startLedgerId; long endLedgerId; - Map delayedIndexBitMap; + Map delayedIndexBitMap; long numberBucketDelayedMessages; @@ -52,24 +52,24 @@ public class BucketState { this(startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, -1, null); } - boolean containsMessage(long ledgerId, int entryId) { - BitSet bitSet = delayedIndexBitMap.get(ledgerId); + boolean containsMessage(long ledgerId, long entryId) { + RoaringBitmap bitSet = delayedIndexBitMap.get(ledgerId); if (bitSet == null) { return false; } - return bitSet.get(entryId); + return bitSet.contains(entryId, entryId + 1); } void putIndexBit(long ledgerId, long entryId) { - delayedIndexBitMap.computeIfAbsent(ledgerId, k -> new BitSet()).set((int) entryId); + delayedIndexBitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); } - boolean removeIndexBit(long ledgerId, int entryId) { + boolean removeIndexBit(long ledgerId, long entryId) { boolean contained = false; - BitSet bitSet = delayedIndexBitMap.get(ledgerId); - if (bitSet != null && bitSet.get(entryId)) { + RoaringBitmap bitSet = delayedIndexBitMap.get(ledgerId); + if (bitSet != null && bitSet.contains(entryId, entryId + 1)) { contained = true; - bitSet.clear(entryId); + bitSet.remove(entryId, entryId + 1); if (bitSet.isEmpty()) { delayedIndexBitMap.remove(ledgerId); diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto index eda9a8f92e670..8414a583fe5b0 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto @@ -24,8 +24,8 @@ option optimize_for = SPEED; message DelayedIndex { required uint64 timestamp = 1; - required int64 ledger_id = 2; - required int64 entry_id = 3; + required uint64 ledger_id = 2; + required uint64 entry_id = 3; } message SnapshotSegmentMetadata { From 270a9c97267de96a5f70244c450432f89b40ceb5 Mon Sep 17 00:00:00 2001 From: coderzc Date: Thu, 13 Oct 2022 11:52:09 +0800 Subject: [PATCH 09/18] Address comment --- .../delayed/BucketDelayedDeliveryTracker.java | 36 +++++++++---------- .../pulsar/broker/delayed/BucketState.java | 15 ++++++-- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 8003ebd23ceb2..4dd06ad45f2a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -49,7 +49,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; @@ -143,6 +142,9 @@ private void moveScheduledMessageToSharedQueue(long cutoffTime) { @Override public void run(Timeout timeout) throws Exception { synchronized (this) { + if (timeout == null || timeout.isCancelled()) { + return; + } moveScheduledMessageToSharedQueue(getCutoffTime()); } super.run(timeout); @@ -160,23 +162,19 @@ private Optional findBucket(long ledgerId) { return Optional.ofNullable(immutableBuckets.get(ledgerId)); } - private Long getBucketId(BucketState bucketState) { - long bucketId = bucketState.getBucketId(); - if (bucketId != -1L) { - return bucketId; + private long getBucketId(BucketState bucketState) { + Optional bucketIdOptional = bucketState.getBucketId(); + if (bucketIdOptional.isPresent()) { + return bucketIdOptional.get(); } String bucketIdStr = cursor.getCursorProperties().get(bucketState.bucketKey()); - if (StringUtils.isBlank(bucketIdStr)) { - return -1L; - } - - bucketId = Long.parseLong(bucketIdStr); + long bucketId = Long.parseLong(bucketIdStr); bucketState.setBucketId(bucketId); return bucketId; } - private BucketState createImmutableBucket(long startLedgerId, long endLedgerId) { + private BucketState createBucket(long startLedgerId, long endLedgerId) { BucketState bucketState = new BucketState(startLedgerId, endLedgerId); immutableBuckets.put(Range.closed(startLedgerId, endLedgerId), bucketState); return bucketState; @@ -276,7 +274,7 @@ private CompletableFuture asyncCreateBucketSnapshot() { final int lastSegmentEntryId = segmentMetadataList.size(); - BucketState bucketState = this.createImmutableBucket(startLedgerId, endLedgerId); + BucketState bucketState = this.createBucket(startLedgerId, endLedgerId); bucketState.setCurrentSegmentEntryId(1); bucketState.setNumberBucketDelayedMessages(numMessages); bucketState.setLastSegmentEntryId(lastSegmentEntryId); @@ -316,13 +314,11 @@ private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState buc } // Wait bucket snapshot create finish - CompletableFuture snapshotCreateFuture = bucketState.snapshotCreateFuture; - if (snapshotCreateFuture == null) { - snapshotCreateFuture = CompletableFuture.completedFuture(-1L); - } + CompletableFuture snapshotCreateFuture = + bucketState.getSnapshotCreateFuture().orElseGet(() -> CompletableFuture.completedFuture(-1L)); return snapshotCreateFuture.thenCompose(__ -> { - final Long bucketId = getBucketId(bucketState); + final long bucketId = getBucketId(bucketState); CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); if (isRecover) { // TODO Recover bucket snapshot @@ -534,8 +530,8 @@ private void cleanImmutableBuckets(boolean delete) { if (bucketState.delayedIndexBitMap != null) { bucketState.delayedIndexBitMap.clear(); } - CompletableFuture snapshotGenerateFuture = bucketState.snapshotCreateFuture; - if (snapshotGenerateFuture != null) { + + bucketState.getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> { if (delete) { snapshotGenerateFuture.cancel(true); // TODO delete bucket snapshot @@ -546,7 +542,7 @@ private void cleanImmutableBuckets(boolean delete) { log.warn("Failed wait to snapshot generate, bucketState: {}", bucketState); } } - } + }); iterator.remove(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java index b42375cbb53ae..bc0d05a5ac8d6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELIMITER; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import lombok.AllArgsConstructor; import lombok.Data; @@ -44,12 +45,12 @@ public class BucketState { long snapshotLength; - long bucketId; + private volatile Long bucketId; - volatile CompletableFuture snapshotCreateFuture; + private volatile CompletableFuture snapshotCreateFuture; BucketState(long startLedgerId, long endLedgerId) { - this(startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, -1, null); + this(startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); } boolean containsMessage(long ledgerId, long entryId) { @@ -86,4 +87,12 @@ public String bucketKey() { return String.join(DELIMITER, DELAYED_BUCKET_KEY_PREFIX, String.valueOf(startLedgerId), String.valueOf(endLedgerId)); } + + public Optional> getSnapshotCreateFuture() { + return Optional.ofNullable(snapshotCreateFuture); + } + + public Optional getBucketId() { + return Optional.ofNullable(bucketId); + } } From 506e8a8f606acc7706796c8c73abd464233f5092 Mon Sep 17 00:00:00 2001 From: coderzc Date: Thu, 13 Oct 2022 18:39:13 +0800 Subject: [PATCH 10/18] Make `putBucketKeyId` run sequentially. --- .../broker/delayed/BucketDelayedDeliveryTracker.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 4dd06ad45f2a7..801c6a68c6fc8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -185,14 +185,13 @@ private CompletableFuture asyncSaveBucketSnapshot( List bucketSnapshotSegments) { return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments) - .thenApply(newBucketId -> { + .thenCompose(newBucketId -> { bucketState.setBucketId(newBucketId); String bucketKey = bucketState.bucketKey(); - putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { + return putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { log.warn("Failed to record bucketId to cursor property, bucketKey: {}", bucketKey); return null; - }); - return newBucketId; + }).thenApply(__ -> newBucketId); }); } @@ -480,6 +479,7 @@ public synchronized Set getScheduledMessages(int maxMessages) { log.debug("[{}] Load next snapshot segment, bucketState: {}", dispatcher.getName(), bucketState); } // All message of current snapshot segment are scheduled, load next snapshot segment + // TODO make it asynchronous try { asyncLoadNextBucketSnapshotEntry(bucketState, false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); From 779f633ca9a04bc78254f3ecf9ce398e9586ddfa Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 18 Oct 2022 21:42:04 +0800 Subject: [PATCH 11/18] Address comment --- .../pulsar/broker/delayed/BucketDelayedDeliveryTracker.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 801c6a68c6fc8..efd779a8370ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -155,10 +155,6 @@ private Optional findBucket(long ledgerId) { return Optional.empty(); } - Range span = immutableBuckets.span(); - if (!span.contains(ledgerId)) { - return Optional.empty(); - } return Optional.ofNullable(immutableBuckets.get(ledgerId)); } @@ -479,7 +475,7 @@ public synchronized Set getScheduledMessages(int maxMessages) { log.debug("[{}] Load next snapshot segment, bucketState: {}", dispatcher.getName(), bucketState); } // All message of current snapshot segment are scheduled, load next snapshot segment - // TODO make it asynchronous + // TODO make it asynchronous and not blocking this process try { asyncLoadNextBucketSnapshotEntry(bucketState, false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); From 642635e1e3a2aab063cdde422dddb250f7dd577b Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 24 Oct 2022 12:01:11 +0800 Subject: [PATCH 12/18] Address comment --- .../delayed/BucketDelayedDeliveryTracker.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index efd779a8370ec..1923296d69f73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -300,6 +300,12 @@ private CompletableFuture asyncCreateBucketSnapshot() { } + /** + * Asynchronous load next bucket snapshot entry. + * @param bucketState bucket state + * @param isRecover whether used to recover bucket snapshot + * @return CompletableFuture + */ private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState bucketState, boolean isRecover) { if (log.isDebugEnabled()) { log.debug("[{}] Load next bucket snapshot data, bucketState: {}", dispatcher.getName(), bucketState); @@ -309,8 +315,9 @@ private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState buc } // Wait bucket snapshot create finish - CompletableFuture snapshotCreateFuture = - bucketState.getSnapshotCreateFuture().orElseGet(() -> CompletableFuture.completedFuture(-1L)); + CompletableFuture snapshotCreateFuture = + bucketState.getSnapshotCreateFuture().orElseGet(() -> CompletableFuture.completedFuture(null)) + .thenApply(__ -> null); return snapshotCreateFuture.thenCompose(__ -> { final long bucketId = getBucketId(bucketState); @@ -415,10 +422,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver public synchronized boolean hasMessageAvailable() { long cutoffTime = getCutoffTime(); - boolean hasMessageAvailable = !getPriorityQueue().isEmpty() && getPriorityQueue().peekN1() <= cutoffTime; - - hasMessageAvailable = hasMessageAvailable - || !sharedBucketPriorityQueue.isEmpty() && sharedBucketPriorityQueue.peekN1() <= cutoffTime; + boolean hasMessageAvailable = getNumberOfDelayedMessages() > 0 && nextDeliveryTime() <= cutoffTime; if (!hasMessageAvailable) { updateTimer(); } From d8110590c83cefec7c697d0d2cbe94359f3c541c Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 24 Oct 2022 16:28:36 +0800 Subject: [PATCH 13/18] Address comment --- .../pulsar/broker/delayed/BucketDelayedDeliveryTracker.java | 2 +- ...ryTrackerTest.java => BucketDelayedDeliveryTrackerTest.java} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/{BuketDelayedDeliveryTrackerTest.java => BucketDelayedDeliveryTrackerTest.java} (98%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 1923296d69f73..9d2e0dd7dd114 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -379,7 +379,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver // Create bucket snapshot if (ledgerId > lastMutableBucketState.endLedgerId && !getPriorityQueue().isEmpty()) { - if (getPriorityQueue().size() >= minIndexCountPerBucket || existBucket) { + if (getPriorityQueue().size() >= minIndexCountPerBucket && !existBucket) { asyncCreateBucketSnapshot(); resetLastMutableBucketRange(); if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java similarity index 98% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java index 9525ba6b72aa8..02b0af54b5ba6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BuketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java @@ -44,7 +44,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class BuketDelayedDeliveryTrackerTest extends InMemoryDeliveryTrackerTest { +public class BucketDelayedDeliveryTrackerTest extends InMemoryDeliveryTrackerTest { private final Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-bucket-delayed-delivery-test"), 500, TimeUnit.MILLISECONDS); From 337c7f2e419ba52c5af7e855478fceced1b231de Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 24 Oct 2022 18:53:22 +0800 Subject: [PATCH 14/18] Address comment --- .../org/apache/bookkeeper/mledger/util/Futures.java | 7 ++++--- .../mledger/impl/ManagedCursorPropertiesTest.java | 6 +++--- .../broker/delayed/BucketDelayedDeliveryTracker.java | 12 +++++------- .../apache/pulsar/broker/delayed/BucketState.java | 6 ++++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java index 637de8a79fc1f..dc1d1eb6c9ac5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java @@ -71,14 +71,15 @@ public static CompletableFuture waitForAll(List> f } public static CompletableFuture executeWithRetry(Supplier> op, - Class needRetryExceptionClass) { + Class needRetryExceptionClass, + int maxRetryTimes) { CompletableFuture resultFuture = new CompletableFuture<>(); op.get().whenComplete((res, ex) -> { if (ex == null) { resultFuture.complete(res); } else { - if (needRetryExceptionClass.isAssignableFrom(ex.getClass())) { - executeWithRetry(op, needRetryExceptionClass).whenComplete((res2, ex2) -> { + if (needRetryExceptionClass.isAssignableFrom(ex.getClass()) && maxRetryTimes > 0) { + executeWithRetry(op, needRetryExceptionClass, maxRetryTimes - 1).whenComplete((res2, ex2) -> { if (ex2 == null) { resultFuture.complete(res2); } else { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java index f0d4083565657..2ef8f3eae327c 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java @@ -247,13 +247,13 @@ public void testUpdateCursorPropertiesConcurrent() throws Exception { map.put("c", "3"); futures.add(executeWithRetry(() -> c1.setCursorProperties(map), - ManagedLedgerException.BadVersionException.class)); + ManagedLedgerException.BadVersionException.class, 3)); futures.add(executeWithRetry(() -> c1.putCursorProperty("a", "2"), - ManagedLedgerException.BadVersionException.class)); + ManagedLedgerException.BadVersionException.class, 3)); futures.add(executeWithRetry(() -> c1.removeCursorProperty("c"), - ManagedLedgerException.BadVersionException.class)); + ManagedLedgerException.BadVersionException.class, 3)); for (CompletableFuture future : futures) { future.get(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 9d2e0dd7dd114..e9f732caf952f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -62,10 +62,7 @@ public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { protected static final int AsyncOperationTimeoutSeconds = 30; - - public static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket"; - - public static final String DELIMITER = "_"; + protected static final int MaxRetryTimes = 3; private final long minIndexCountPerBucket; @@ -194,10 +191,10 @@ private CompletableFuture asyncSaveBucketSnapshot( private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { Objects.requireNonNull(bucketId); return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), - ManagedLedgerException.BadVersionException.class); + ManagedLedgerException.BadVersionException.class, MaxRetryTimes); } - private CompletableFuture asyncCreateBucketSnapshot() { + private CompletableFuture createBucketSnapshotAndAsyncPersistent() { TripleLongPriorityQueue priorityQueue = super.getPriorityQueue(); if (priorityQueue.isEmpty()) { return CompletableFuture.completedFuture(-1L); @@ -380,7 +377,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver // Create bucket snapshot if (ledgerId > lastMutableBucketState.endLedgerId && !getPriorityQueue().isEmpty()) { if (getPriorityQueue().size() >= minIndexCountPerBucket && !existBucket) { - asyncCreateBucketSnapshot(); + createBucketSnapshotAndAsyncPersistent(); resetLastMutableBucketRange(); if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { // TODO merge bucket snapshot (synchronize operate) @@ -484,6 +481,7 @@ public synchronized Set getScheduledMessages(int maxMessages) { asyncLoadNextBucketSnapshotEntry(bucketState, false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { + // TODO make this segment load again throw new RuntimeException(e); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java index bc0d05a5ac8d6..2ff3ce65a708b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.broker.delayed; -import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; -import static org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTracker.DELIMITER; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -32,6 +30,10 @@ @AllArgsConstructor public class BucketState { + public static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket"; + + public static final String DELIMITER = "_"; + long startLedgerId; long endLedgerId; From 31efe2e01d870bfe55adcd44e3b11eed2c50eed1 Mon Sep 17 00:00:00 2001 From: coderzc Date: Fri, 28 Oct 2022 13:25:09 +0800 Subject: [PATCH 15/18] Remove `highestDeliveryTimeTracked`/`messagesHaveFixedDelay`/`highestDeliveryTimeTracked` in BucketDelayedDeliveryTracker. --- .../delayed/BucketDelayedDeliveryTracker.java | 20 +++------ .../InMemoryDelayedDeliveryTracker.java | 6 +-- .../BucketDelayedDeliveryTrackerTest.java | 41 +++++++++++++------ 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index e9f732caf952f..5714d765a118e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -87,24 +87,21 @@ public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, - long fixedDelayDetectionLookahead, BucketSnapshotStorage bucketSnapshotStorage, long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, int maxNumBuckets) { this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, - fixedDelayDetectionLookahead, bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegment, maxNumBuckets); } BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict, - long fixedDelayDetectionLookahead, BucketSnapshotStorage bucketSnapshotStorage, long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, int maxNumBuckets) { super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict, - fixedDelayDetectionLookahead); + -1L); this.minIndexCountPerBucket = minIndexCountPerBucket; this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; this.maxNumBuckets = maxNumBuckets; @@ -363,12 +360,10 @@ private void resetLastMutableBucketRange() { @Override public synchronized boolean addMessage(long ledgerId, long entryId, long deliverAt) { if (containsMessage(ledgerId, entryId)) { - messagesHaveFixedDelay = false; return true; } if (deliverAt < 0 || deliverAt <= getCutoffTime()) { - messagesHaveFixedDelay = false; return false; } @@ -410,8 +405,6 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver updateTimer(); - checkAndUpdateHighest(deliverAt); - return true; } @@ -490,17 +483,16 @@ public synchronized Set getScheduledMessages(int maxMessages) { --numberDelayedMessages; } - if (numberDelayedMessages <= 0) { - // Reset to initial state - highestDeliveryTimeTracked = 0; - messagesHaveFixedDelay = true; - } - updateTimer(); return positions; } + @Override + public boolean shouldPauseAllDeliveries() { + return false; + } + @Override public synchronized void clear() { super.clear(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index c7db10157f1d7..9b2d485f45b7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -68,10 +68,10 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T // This is the timestamp of the message with the highest delivery time // If new added messages are lower than this, it means the delivery is requested // to be out-of-order. It gets reset to 0, once the tracker is emptied. - protected long highestDeliveryTimeTracked = 0; + private long highestDeliveryTimeTracked = 0; // Track whether we have seen all messages with fixed delay so far. - protected boolean messagesHaveFixedDelay = true; + private boolean messagesHaveFixedDelay = true; InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -132,7 +132,7 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { * Check that new delivery time comes after the current highest, or at * least within a single tick time interval of 1 second. */ - protected void checkAndUpdateHighest(long deliverAt) { + private void checkAndUpdateHighest(long deliverAt) { if (deliverAt < (highestDeliveryTimeTracked - tickTimeMillis)) { messagesHaveFixedDelay = false; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java index 02b0af54b5ba6..f7525c5b8eee1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java @@ -75,7 +75,7 @@ public Object[][] provider(Method method) throws Exception { return switch (methodName) { case "test" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testWithTimer" -> { Timer timer = mock(Timer.class); @@ -103,39 +103,35 @@ public Object[][] provider(Method method) throws Exception { yield new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50), + false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50), tasks }}; } case "testAddWithinTickTime" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock, - false, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testAddMessageWithStrictDelay" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock, - true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, - true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict", "testRecoverSnapshot" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, - true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict", "testExistDelayedMessage" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, - true, 0, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; - case "testWithFixedDelays", "testWithMixedDelays", "testWithNoDelays" -> new Object[][]{{ - new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, - true, 100, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) - }}; default -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true, 0, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), 50) + true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), 50) }}; }; } @@ -163,4 +159,25 @@ public void testContainsMessage(DelayedDeliveryTracker tracker) { tracker.close(); } + + @Override + @Test(dataProvider = "delayedTracker") + public void testWithFixedDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { + assertEquals(tracker.getFixedDelayDetectionLookahead(), -1L); + tracker.close(); + } + + @Override + @Test(dataProvider = "delayedTracker") + public void testWithMixedDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { + assertEquals(tracker.getFixedDelayDetectionLookahead(), -1L); + tracker.close(); + } + + @Override + @Test(dataProvider = "delayedTracker") + public void testWithNoDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { + assertEquals(tracker.getFixedDelayDetectionLookahead(), -1L); + tracker.close(); + } } From 99fd20bc8ec8a0a65273402e1ea3192dcdd2cab6 Mon Sep 17 00:00:00 2001 From: coderzc Date: Sat, 29 Oct 2022 14:23:07 +0800 Subject: [PATCH 16/18] fix style --- .../broker/delayed/BucketDelayedDeliveryTracker.java | 8 ++++---- .../org/apache/pulsar/broker/delayed/BucketState.java | 2 +- .../broker/delayed/BucketDelayedDeliveryTrackerTest.java | 2 +- .../pulsar/broker/delayed/MockBucketSnapshotStorage.java | 2 +- .../apache/pulsar/broker/delayed/MockManagedCursor.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java index 5714d765a118e..0a0587e671d71 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -61,8 +61,8 @@ @ThreadSafe public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { - protected static final int AsyncOperationTimeoutSeconds = 30; - protected static final int MaxRetryTimes = 3; + private static final int AsyncOperationTimeoutSeconds = 30; + private static final int MaxRetryTimes = 3; private final long minIndexCountPerBucket; @@ -72,7 +72,7 @@ public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker private final ManagedCursor cursor; - public final BucketSnapshotStorage bucketSnapshotStorage; + private final BucketSnapshotStorage bucketSnapshotStorage; private long numberDelayedMessages; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java index 2ff3ce65a708b..c00e7ef8d4e12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java index f7525c5b8eee1..3a482ffb1ed56 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index 352fea65be909..23b5c9b7e0817 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index 945eb183ec7e4..efb0fa7ab7ba2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From ca7b00cafde2c168eeb067f0718a44fa4f34c826 Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 1 Nov 2022 23:07:56 +0800 Subject: [PATCH 17/18] Refactor Bucket --- .../delayed/BucketDelayedDeliveryTracker.java | 559 ------------------ .../InMemoryDelayedDeliveryTracker.java | 2 +- .../{BucketState.java => bucket/Bucket.java} | 63 +- .../bucket/BucketDelayedDeliveryTracker.java | 349 +++++++++++ .../{ => bucket}/BucketSnapshotStorage.java | 2 +- .../delayed/bucket/ImmutableBucket.java | 101 ++++ .../broker/delayed/bucket/MutableBucket.java | 159 +++++ .../broker/delayed/bucket/package-info.java | 19 + .../BucketDelayedDeliveryTrackerTest.java | 2 + .../delayed/MockBucketSnapshotStorage.java | 1 + 10 files changed, 687 insertions(+), 570 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java rename pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/{BucketState.java => bucket/Bucket.java} (50%) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java rename pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/{ => bucket}/BucketSnapshotStorage.java (98%) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java deleted file mode 100644 index 0a0587e671d71..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTracker.java +++ /dev/null @@ -1,559 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.delayed; - -import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Range; -import com.google.common.collect.RangeMap; -import com.google.common.collect.Table; -import com.google.common.collect.TreeRangeMap; -import com.google.protobuf.ByteString; -import io.netty.util.Timeout; -import io.netty.util.Timer; -import java.nio.ByteBuffer; -import java.time.Clock; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import javax.annotation.concurrent.ThreadSafe; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; -import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; -import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; -import org.roaringbitmap.RoaringBitmap; - -@Slf4j -@ThreadSafe -public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { - - private static final int AsyncOperationTimeoutSeconds = 30; - private static final int MaxRetryTimes = 3; - - private final long minIndexCountPerBucket; - - private final long timeStepPerBucketSnapshotSegment; - - private final int maxNumBuckets; - - private final ManagedCursor cursor; - - private final BucketSnapshotStorage bucketSnapshotStorage; - - private long numberDelayedMessages; - - private final BucketState lastMutableBucketState; - - private final TripleLongPriorityQueue sharedBucketPriorityQueue; - - private final RangeMap immutableBuckets; - - private final Table snapshotSegmentLastIndexTable; - - BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, - Timer timer, long tickTimeMillis, - boolean isDelayedDeliveryDeliverAtTimeStrict, - BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, - int maxNumBuckets) { - this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, - bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegment, maxNumBuckets); - } - - BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, - Timer timer, long tickTimeMillis, Clock clock, - boolean isDelayedDeliveryDeliverAtTimeStrict, - BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, - int maxNumBuckets) { - super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict, - -1L); - this.minIndexCountPerBucket = minIndexCountPerBucket; - this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; - this.maxNumBuckets = maxNumBuckets; - this.cursor = dispatcher.getCursor(); - this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); - this.immutableBuckets = TreeRangeMap.create(); - this.snapshotSegmentLastIndexTable = HashBasedTable.create(); - - this.bucketSnapshotStorage = bucketSnapshotStorage; - - this.numberDelayedMessages = 0L; - - this.lastMutableBucketState = new BucketState(-1L, -1L); - } - - private void moveScheduledMessageToSharedQueue(long cutoffTime) { - TripleLongPriorityQueue priorityQueue = getPriorityQueue(); - while (!priorityQueue.isEmpty()) { - long timestamp = priorityQueue.peekN1(); - if (timestamp > cutoffTime) { - break; - } - - long ledgerId = priorityQueue.peekN2(); - long entryId = priorityQueue.peekN3(); - sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); - - priorityQueue.pop(); - } - } - - @Override - public void run(Timeout timeout) throws Exception { - synchronized (this) { - if (timeout == null || timeout.isCancelled()) { - return; - } - moveScheduledMessageToSharedQueue(getCutoffTime()); - } - super.run(timeout); - } - - private Optional findBucket(long ledgerId) { - if (immutableBuckets.asMapOfRanges().isEmpty()) { - return Optional.empty(); - } - - return Optional.ofNullable(immutableBuckets.get(ledgerId)); - } - - private long getBucketId(BucketState bucketState) { - Optional bucketIdOptional = bucketState.getBucketId(); - if (bucketIdOptional.isPresent()) { - return bucketIdOptional.get(); - } - - String bucketIdStr = cursor.getCursorProperties().get(bucketState.bucketKey()); - long bucketId = Long.parseLong(bucketIdStr); - bucketState.setBucketId(bucketId); - return bucketId; - } - - private BucketState createBucket(long startLedgerId, long endLedgerId) { - BucketState bucketState = new BucketState(startLedgerId, endLedgerId); - immutableBuckets.put(Range.closed(startLedgerId, endLedgerId), bucketState); - return bucketState; - } - - private CompletableFuture asyncSaveBucketSnapshot( - BucketState bucketState, SnapshotMetadata snapshotMetadata, - List bucketSnapshotSegments) { - - return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments) - .thenCompose(newBucketId -> { - bucketState.setBucketId(newBucketId); - String bucketKey = bucketState.bucketKey(); - return putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { - log.warn("Failed to record bucketId to cursor property, bucketKey: {}", bucketKey); - return null; - }).thenApply(__ -> newBucketId); - }); - } - - private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { - Objects.requireNonNull(bucketId); - return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), - ManagedLedgerException.BadVersionException.class, MaxRetryTimes); - } - - private CompletableFuture createBucketSnapshotAndAsyncPersistent() { - TripleLongPriorityQueue priorityQueue = super.getPriorityQueue(); - if (priorityQueue.isEmpty()) { - return CompletableFuture.completedFuture(-1L); - } - long numMessages = 0; - - final long startLedgerId = lastMutableBucketState.startLedgerId; - final long endLedgerId = lastMutableBucketState.endLedgerId; - - List bucketSnapshotSegments = new ArrayList<>(); - List segmentMetadataList = new ArrayList<>(); - Map bitMap = new HashMap<>(); - SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); - SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); - - long currentTimestampUpperLimit = 0; - while (!priorityQueue.isEmpty()) { - long timestamp = priorityQueue.peekN1(); - if (currentTimestampUpperLimit == 0) { - currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1; - } - - long ledgerId = priorityQueue.peekN2(); - long entryId = priorityQueue.peekN3(); - - checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); - - // Move first segment of bucket snapshot to sharedBucketPriorityQueue - if (segmentMetadataList.size() == 0) { - sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); - } - - priorityQueue.pop(); - numMessages++; - - DelayedIndex delayedIndex = DelayedIndex.newBuilder() - .setTimestamp(timestamp) - .setLedgerId(ledgerId) - .setEntryId(entryId).build(); - - bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); - - snapshotSegmentBuilder.addIndexes(delayedIndex); - - if (priorityQueue.isEmpty() || priorityQueue.peekN1() > currentTimestampUpperLimit) { - segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); - currentTimestampUpperLimit = 0; - - Iterator> iterator = bitMap.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - byte[] array = new byte[entry.getValue().serializedSizeInBytes()]; - entry.getValue().serialize(ByteBuffer.wrap(array)); - segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array)); - iterator.remove(); - } - - segmentMetadataList.add(segmentMetadataBuilder.build()); - segmentMetadataBuilder.clear(); - - bucketSnapshotSegments.add(snapshotSegmentBuilder.build()); - snapshotSegmentBuilder.clear(); - } - } - - SnapshotMetadata bucketSnapshotMetadata = SnapshotMetadata.newBuilder() - .addAllMetadataList(segmentMetadataList) - .build(); - - final int lastSegmentEntryId = segmentMetadataList.size(); - - BucketState bucketState = this.createBucket(startLedgerId, endLedgerId); - bucketState.setCurrentSegmentEntryId(1); - bucketState.setNumberBucketDelayedMessages(numMessages); - bucketState.setLastSegmentEntryId(lastSegmentEntryId); - - // Add the first snapshot segment last message to snapshotSegmentLastMessageTable - checkArgument(!bucketSnapshotSegments.isEmpty()); - SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - DelayedIndex delayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); - snapshotSegmentLastIndexTable.put(delayedIndex.getLedgerId(), delayedIndex.getEntryId(), bucketState); - - if (log.isDebugEnabled()) { - log.debug("[{}] Create bucket snapshot, bucketState: {}", dispatcher.getName(), bucketState); - } - - CompletableFuture future = asyncSaveBucketSnapshot(bucketState, - bucketSnapshotMetadata, bucketSnapshotSegments); - bucketState.setSnapshotCreateFuture(future); - future.whenComplete((__, ex) -> { - if (ex == null) { - bucketState.setSnapshotCreateFuture(null); - } else { - //TODO Record create snapshot failed - log.error("Failed to create snapshot: ", ex); - } - }); - - return future; - } - - - /** - * Asynchronous load next bucket snapshot entry. - * @param bucketState bucket state - * @param isRecover whether used to recover bucket snapshot - * @return CompletableFuture - */ - private CompletableFuture asyncLoadNextBucketSnapshotEntry(BucketState bucketState, boolean isRecover) { - if (log.isDebugEnabled()) { - log.debug("[{}] Load next bucket snapshot data, bucketState: {}", dispatcher.getName(), bucketState); - } - if (bucketState == null) { - return CompletableFuture.completedFuture(null); - } - - // Wait bucket snapshot create finish - CompletableFuture snapshotCreateFuture = - bucketState.getSnapshotCreateFuture().orElseGet(() -> CompletableFuture.completedFuture(null)) - .thenApply(__ -> null); - - return snapshotCreateFuture.thenCompose(__ -> { - final long bucketId = getBucketId(bucketState); - CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); - if (isRecover) { - // TODO Recover bucket snapshot - } else { - loadMetaDataFuture.complete(bucketState.currentSegmentEntryId + 1); - } - - return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { - if (nextSegmentEntryId > bucketState.lastSegmentEntryId) { - // TODO Delete bucket snapshot - return CompletableFuture.completedFuture(null); - } - - return bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId) - .thenAccept(bucketSnapshotSegments -> { - if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { - return; - } - - SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - List indexList = snapshotSegment.getIndexesList(); - DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); - - this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), - lastDelayedIndex.getEntryId(), bucketState); - - for (DelayedIndex index : indexList) { - sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), - index.getEntryId()); - } - - bucketState.setCurrentSegmentEntryId(nextSegmentEntryId); - }); - }); - }); - } - - private void resetLastMutableBucketRange() { - lastMutableBucketState.setStartLedgerId(-1L); - lastMutableBucketState.setEndLedgerId(-1L); - } - - @Override - public synchronized boolean addMessage(long ledgerId, long entryId, long deliverAt) { - if (containsMessage(ledgerId, entryId)) { - return true; - } - - if (deliverAt < 0 || deliverAt <= getCutoffTime()) { - return false; - } - - boolean existBucket = findBucket(ledgerId).isPresent(); - - // Create bucket snapshot - if (ledgerId > lastMutableBucketState.endLedgerId && !getPriorityQueue().isEmpty()) { - if (getPriorityQueue().size() >= minIndexCountPerBucket && !existBucket) { - createBucketSnapshotAndAsyncPersistent(); - resetLastMutableBucketRange(); - if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { - // TODO merge bucket snapshot (synchronize operate) - } - } - } - - if (ledgerId < lastMutableBucketState.startLedgerId || existBucket) { - // If (ledgerId < startLedgerId || existBucket) means that message index belong to previous bucket range, - // enter sharedBucketPriorityQueue directly - sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); - } else { - checkArgument(ledgerId >= lastMutableBucketState.endLedgerId); - - getPriorityQueue().add(deliverAt, ledgerId, entryId); - - if (lastMutableBucketState.startLedgerId == -1L) { - lastMutableBucketState.setStartLedgerId(ledgerId); - } - lastMutableBucketState.setEndLedgerId(ledgerId); - } - - lastMutableBucketState.putIndexBit(ledgerId, entryId); - numberDelayedMessages++; - - if (log.isDebugEnabled()) { - log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", dispatcher.getName(), ledgerId, entryId, - deliverAt - clock.millis()); - } - - updateTimer(); - - return true; - } - - @Override - public synchronized boolean hasMessageAvailable() { - long cutoffTime = getCutoffTime(); - - boolean hasMessageAvailable = getNumberOfDelayedMessages() > 0 && nextDeliveryTime() <= cutoffTime; - if (!hasMessageAvailable) { - updateTimer(); - } - return hasMessageAvailable; - } - - @Override - protected long nextDeliveryTime() { - if (getPriorityQueue().isEmpty() && !sharedBucketPriorityQueue.isEmpty()) { - return sharedBucketPriorityQueue.peekN1(); - } else if (sharedBucketPriorityQueue.isEmpty() && !getPriorityQueue().isEmpty()) { - return getPriorityQueue().peekN1(); - } - long timestamp = getPriorityQueue().peekN1(); - long bucketTimestamp = sharedBucketPriorityQueue.peekN1(); - return Math.min(timestamp, bucketTimestamp); - } - - @Override - public synchronized long getNumberOfDelayedMessages() { - return numberDelayedMessages; - } - - @Override - public synchronized long getBufferMemoryUsage() { - return getPriorityQueue().bytesCapacity() + sharedBucketPriorityQueue.bytesCapacity(); - } - - @Override - public synchronized Set getScheduledMessages(int maxMessages) { - long cutoffTime = getCutoffTime(); - - moveScheduledMessageToSharedQueue(cutoffTime); - - Set positions = new TreeSet<>(); - int n = maxMessages; - - while (n > 0 && !sharedBucketPriorityQueue.isEmpty()) { - long timestamp = sharedBucketPriorityQueue.peekN1(); - if (timestamp > cutoffTime) { - break; - } - - long ledgerId = sharedBucketPriorityQueue.peekN2(); - long entryId = sharedBucketPriorityQueue.peekN3(); - positions.add(new PositionImpl(ledgerId, entryId)); - - sharedBucketPriorityQueue.pop(); - removeIndexBit(ledgerId, entryId); - - BucketState bucketState = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); - if (bucketState != null) { - if (log.isDebugEnabled()) { - log.debug("[{}] Load next snapshot segment, bucketState: {}", dispatcher.getName(), bucketState); - } - // All message of current snapshot segment are scheduled, load next snapshot segment - // TODO make it asynchronous and not blocking this process - try { - asyncLoadNextBucketSnapshotEntry(bucketState, false).get(AsyncOperationTimeoutSeconds, - TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - // TODO make this segment load again - throw new RuntimeException(e); - } - } - - --n; - --numberDelayedMessages; - } - - updateTimer(); - - return positions; - } - - @Override - public boolean shouldPauseAllDeliveries() { - return false; - } - - @Override - public synchronized void clear() { - super.clear(); - cleanImmutableBuckets(true); - sharedBucketPriorityQueue.clear(); - resetLastMutableBucketRange(); - lastMutableBucketState.delayedIndexBitMap.clear(); - snapshotSegmentLastIndexTable.clear(); - numberDelayedMessages = 0; - } - - @Override - public synchronized void close() { - super.close(); - cleanImmutableBuckets(false); - lastMutableBucketState.delayedIndexBitMap.clear(); - sharedBucketPriorityQueue.close(); - } - - private void cleanImmutableBuckets(boolean delete) { - if (immutableBuckets != null) { - Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); - while (iterator.hasNext()) { - BucketState bucketState = iterator.next(); - if (bucketState.delayedIndexBitMap != null) { - bucketState.delayedIndexBitMap.clear(); - } - - bucketState.getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> { - if (delete) { - snapshotGenerateFuture.cancel(true); - // TODO delete bucket snapshot - } else { - try { - snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (Exception e) { - log.warn("Failed wait to snapshot generate, bucketState: {}", bucketState); - } - } - }); - iterator.remove(); - } - } - } - - private boolean removeIndexBit(long ledgerId, long entryId) { - if (lastMutableBucketState.removeIndexBit(ledgerId, entryId)) { - return true; - } - - return findBucket(ledgerId).map(bucketState -> bucketState.removeIndexBit(ledgerId, entryId)) - .orElse(false); - } - - @Override - public boolean containsMessage(long ledgerId, long entryId) { - if (lastMutableBucketState.containsMessage(ledgerId, entryId)) { - return true; - } - - return findBucket(ledgerId).map(bucketState -> bucketState.containsMessage(ledgerId, entryId)) - .orElse(false); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 9b2d485f45b7e..58c86deb410e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -80,7 +80,7 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T fixedDelayDetectionLookahead); } - InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, + public InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict, long fixedDelayDetectionLookahead) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java similarity index 50% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index c00e7ef8d4e12..25ff9d033e595 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -16,23 +16,34 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.delayed; +package org.apache.pulsar.broker.delayed.bucket; +import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.roaringbitmap.RoaringBitmap; +@Slf4j @Data @AllArgsConstructor -public class BucketState { +public abstract class Bucket { - public static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket"; + static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket"; + static final String DELIMITER = "_"; + static final int MaxRetryTimes = 3; - public static final String DELIMITER = "_"; + protected final ManagedCursor cursor; + protected final BucketSnapshotStorage bucketSnapshotStorage; long startLedgerId; long endLedgerId; @@ -51,8 +62,9 @@ public class BucketState { private volatile CompletableFuture snapshotCreateFuture; - BucketState(long startLedgerId, long endLedgerId) { - this(startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); + + Bucket(ManagedCursor cursor, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { + this(cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); } boolean containsMessage(long ledgerId, long entryId) { @@ -85,16 +97,49 @@ boolean removeIndexBit(long ledgerId, long entryId) { return contained; } - public String bucketKey() { + String bucketKey() { return String.join(DELIMITER, DELAYED_BUCKET_KEY_PREFIX, String.valueOf(startLedgerId), String.valueOf(endLedgerId)); } - public Optional> getSnapshotCreateFuture() { + Optional> getSnapshotCreateFuture() { return Optional.ofNullable(snapshotCreateFuture); } - public Optional getBucketId() { + Optional getBucketId() { return Optional.ofNullable(bucketId); } + + long getAndUpdateBucketId() { + Optional bucketIdOptional = getBucketId(); + if (bucketIdOptional.isPresent()) { + return bucketIdOptional.get(); + } + + String bucketIdStr = cursor.getCursorProperties().get(bucketKey()); + long bucketId = Long.parseLong(bucketIdStr); + setBucketId(bucketId); + return bucketId; + } + + CompletableFuture asyncSaveBucketSnapshot( + ImmutableBucket bucketState, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata, + List bucketSnapshotSegments) { + + return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments) + .thenCompose(newBucketId -> { + bucketState.setBucketId(newBucketId); + String bucketKey = bucketState.bucketKey(); + return putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { + log.warn("Failed to record bucketId to cursor property, bucketKey: {}", bucketKey); + return null; + }).thenApply(__ -> newBucketId); + }); + } + + private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { + Objects.requireNonNull(bucketId); + return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), + ManagedLedgerException.BadVersionException.class, MaxRetryTimes); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java new file mode 100644 index 0000000000000..71b554fef41d5 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; +import com.google.common.collect.Table; +import com.google.common.collect.TreeRangeMap; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import java.time.Clock; +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.annotation.concurrent.ThreadSafe; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; + +@Slf4j +@ThreadSafe +public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { + + static final int AsyncOperationTimeoutSeconds = 30; + + private final long minIndexCountPerBucket; + + private final long timeStepPerBucketSnapshotSegment; + + private final int maxNumBuckets; + + private long numberDelayedMessages; + + private final MutableBucket lastMutableBucket; + + private final TripleLongPriorityQueue sharedBucketPriorityQueue; + + private final RangeMap immutableBuckets; + + private final Table snapshotSegmentLastIndexTable; + + public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, + Timer timer, long tickTimeMillis, + boolean isDelayedDeliveryDeliverAtTimeStrict, + BucketSnapshotStorage bucketSnapshotStorage, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + int maxNumBuckets) { + this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, + bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegment, maxNumBuckets); + } + + public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, + Timer timer, long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict, + BucketSnapshotStorage bucketSnapshotStorage, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + int maxNumBuckets) { + super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict, + -1L); + this.minIndexCountPerBucket = minIndexCountPerBucket; + this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; + this.maxNumBuckets = maxNumBuckets; + ManagedCursor cursor = dispatcher.getCursor(); + this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); + this.immutableBuckets = TreeRangeMap.create(); + this.snapshotSegmentLastIndexTable = HashBasedTable.create(); + + this.numberDelayedMessages = 0L; + + this.lastMutableBucket = new MutableBucket(cursor, bucketSnapshotStorage, super.getPriorityQueue()); + } + + private void moveScheduledMessageToSharedQueue(long cutoffTime) { + TripleLongPriorityQueue priorityQueue = getPriorityQueue(); + while (!priorityQueue.isEmpty()) { + long timestamp = priorityQueue.peekN1(); + if (timestamp > cutoffTime) { + break; + } + + long ledgerId = priorityQueue.peekN2(); + long entryId = priorityQueue.peekN3(); + sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); + + priorityQueue.pop(); + } + } + + @Override + public void run(Timeout timeout) throws Exception { + synchronized (this) { + if (timeout == null || timeout.isCancelled()) { + return; + } + moveScheduledMessageToSharedQueue(getCutoffTime()); + } + super.run(timeout); + } + + private Optional findImmutableBucket(long ledgerId) { + if (immutableBuckets.asMapOfRanges().isEmpty()) { + return Optional.empty(); + } + + return Optional.ofNullable(immutableBuckets.get(ledgerId)); + } + + private void sealBucket() { + Pair immutableBucketDelayedIndexPair = + lastMutableBucket.sealBucketAndAsyncPersistent(this.timeStepPerBucketSnapshotSegment, + this.sharedBucketPriorityQueue); + if (immutableBucketDelayedIndexPair != null) { + ImmutableBucket immutableBucket = immutableBucketDelayedIndexPair.getLeft(); + immutableBuckets.put(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), + immutableBucket); + + DelayedIndex lastDelayedIndex = immutableBucketDelayedIndexPair.getRight(); + snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId(), + immutableBucket); + if (log.isDebugEnabled()) { + log.debug("[{}] Create bucket snapshot, bucket: {}", dispatcher.getName(), + lastMutableBucket); + } + } + } + + @Override + public synchronized boolean addMessage(long ledgerId, long entryId, long deliverAt) { + if (containsMessage(ledgerId, entryId)) { + return true; + } + + if (deliverAt < 0 || deliverAt <= getCutoffTime()) { + return false; + } + + boolean existBucket = findImmutableBucket(ledgerId).isPresent(); + + // Create bucket snapshot + if (ledgerId > lastMutableBucket.endLedgerId && !getPriorityQueue().isEmpty()) { + if (getPriorityQueue().size() >= minIndexCountPerBucket && !existBucket) { + sealBucket(); + lastMutableBucket.resetLastMutableBucketRange(); + if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { + // TODO merge bucket snapshot (synchronize operate) + } + } + } + + if (ledgerId < lastMutableBucket.startLedgerId || existBucket) { + // If (ledgerId < startLedgerId || existBucket) means that message index belong to previous bucket range, + // enter sharedBucketPriorityQueue directly + sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); + } else { + checkArgument(ledgerId >= lastMutableBucket.endLedgerId); + + getPriorityQueue().add(deliverAt, ledgerId, entryId); + + if (lastMutableBucket.startLedgerId == -1L) { + lastMutableBucket.setStartLedgerId(ledgerId); + } + lastMutableBucket.setEndLedgerId(ledgerId); + } + + lastMutableBucket.putIndexBit(ledgerId, entryId); + numberDelayedMessages++; + + if (log.isDebugEnabled()) { + log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", dispatcher.getName(), ledgerId, entryId, + deliverAt - clock.millis()); + } + + updateTimer(); + + return true; + } + + @Override + public synchronized boolean hasMessageAvailable() { + long cutoffTime = getCutoffTime(); + + boolean hasMessageAvailable = getNumberOfDelayedMessages() > 0 && nextDeliveryTime() <= cutoffTime; + if (!hasMessageAvailable) { + updateTimer(); + } + return hasMessageAvailable; + } + + @Override + protected long nextDeliveryTime() { + if (getPriorityQueue().isEmpty() && !sharedBucketPriorityQueue.isEmpty()) { + return sharedBucketPriorityQueue.peekN1(); + } else if (sharedBucketPriorityQueue.isEmpty() && !getPriorityQueue().isEmpty()) { + return getPriorityQueue().peekN1(); + } + long timestamp = getPriorityQueue().peekN1(); + long bucketTimestamp = sharedBucketPriorityQueue.peekN1(); + return Math.min(timestamp, bucketTimestamp); + } + + @Override + public synchronized long getNumberOfDelayedMessages() { + return numberDelayedMessages; + } + + @Override + public synchronized long getBufferMemoryUsage() { + return getPriorityQueue().bytesCapacity() + sharedBucketPriorityQueue.bytesCapacity(); + } + + @Override + public synchronized Set getScheduledMessages(int maxMessages) { + long cutoffTime = getCutoffTime(); + + moveScheduledMessageToSharedQueue(cutoffTime); + + Set positions = new TreeSet<>(); + int n = maxMessages; + + while (n > 0 && !sharedBucketPriorityQueue.isEmpty()) { + long timestamp = sharedBucketPriorityQueue.peekN1(); + if (timestamp > cutoffTime) { + break; + } + + long ledgerId = sharedBucketPriorityQueue.peekN2(); + long entryId = sharedBucketPriorityQueue.peekN3(); + positions.add(new PositionImpl(ledgerId, entryId)); + + sharedBucketPriorityQueue.pop(); + removeIndexBit(ledgerId, entryId); + + ImmutableBucket bucket = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); + if (bucket != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Load next snapshot segment, bucket: {}", dispatcher.getName(), bucket); + } + // All message of current snapshot segment are scheduled, load next snapshot segment + // TODO make it asynchronous and not blocking this process + try { + bucket.asyncLoadNextBucketSnapshotEntry(false).thenAccept(indexList -> { + if (CollectionUtils.isEmpty(indexList)) { + return; + } + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex + lastDelayedIndex = indexList.get(indexList.size() - 1); + this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), + lastDelayedIndex.getEntryId(), bucket); + for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : indexList) { + sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), + index.getEntryId()); + } + }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + // TODO make this segment load again + throw new RuntimeException(e); + } + } + + --n; + --numberDelayedMessages; + } + + updateTimer(); + + return positions; + } + + @Override + public boolean shouldPauseAllDeliveries() { + return false; + } + + @Override + public synchronized void clear() { + super.clear(); + cleanImmutableBuckets(true); + sharedBucketPriorityQueue.clear(); + lastMutableBucket.clear(); + snapshotSegmentLastIndexTable.clear(); + numberDelayedMessages = 0; + } + + @Override + public synchronized void close() { + super.close(); + cleanImmutableBuckets(false); + sharedBucketPriorityQueue.close(); + } + + private void cleanImmutableBuckets(boolean delete) { + if (immutableBuckets != null) { + Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); + while (iterator.hasNext()) { + ImmutableBucket bucket = iterator.next(); + bucket.clear(delete); + iterator.remove(); + } + } + } + + private boolean removeIndexBit(long ledgerId, long entryId) { + if (lastMutableBucket.removeIndexBit(ledgerId, entryId)) { + return true; + } + + return findImmutableBucket(ledgerId).map(bucket -> bucket.removeIndexBit(ledgerId, entryId)) + .orElse(false); + } + + @Override + public boolean containsMessage(long ledgerId, long entryId) { + if (lastMutableBucket.containsMessage(ledgerId, entryId)) { + return true; + } + + return findImmutableBucket(ledgerId).map(bucket -> bucket.containsMessage(ledgerId, entryId)) + .orElse(false); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java similarity index 98% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketSnapshotStorage.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java index 7e5fa633dd914..3ab4ce1ad2792 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.delayed; +package org.apache.pulsar.broker.delayed.bucket; import java.util.List; import java.util.concurrent.CompletableFuture; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java new file mode 100644 index 0000000000000..833030c575172 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; + +@Slf4j +class ImmutableBucket extends Bucket { + ImmutableBucket(ManagedCursor cursor, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { + super(cursor, storage, startLedgerId, endLedgerId); + } + + /** + * Asynchronous load next bucket snapshot entry. + * @param isRecover whether used to recover bucket snapshot + * @return CompletableFuture + */ + CompletableFuture> asyncLoadNextBucketSnapshotEntry(boolean isRecover) { + if (log.isDebugEnabled()) { + log.debug("[{}] Load next bucket snapshot data, bucket: {}", cursor.getName(), this); + } + + // Wait bucket snapshot create finish + CompletableFuture snapshotCreateFuture = + getSnapshotCreateFuture().orElseGet(() -> CompletableFuture.completedFuture(null)) + .thenApply(__ -> null); + + return snapshotCreateFuture.thenCompose(__ -> { + final long bucketId = getAndUpdateBucketId(); + CompletableFuture loadMetaDataFuture = new CompletableFuture<>(); + if (isRecover) { + // TODO Recover bucket snapshot + } else { + loadMetaDataFuture.complete(currentSegmentEntryId + 1); + } + + return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { + if (nextSegmentEntryId > lastSegmentEntryId) { + // TODO Delete bucket snapshot + return CompletableFuture.completedFuture(null); + } + + return bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId) + .thenApply(bucketSnapshotSegments -> { + if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { + return Collections.emptyList(); + } + + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + bucketSnapshotSegments.get(0); + List indexList = + snapshotSegment.getIndexesList(); + this.setCurrentSegmentEntryId(nextSegmentEntryId); + return indexList; + }); + }); + }); + } + + void clear(boolean delete) { + delayedIndexBitMap.clear(); + getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> { + if (delete) { + snapshotGenerateFuture.cancel(true); + // TODO delete bucket snapshot + } else { + try { + snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); + } catch (Exception e) { + log.warn("Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", getBucketId(), + bucketKey()); + } + } + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java new file mode 100644 index 0000000000000..4673809105b04 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import static com.google.common.base.Preconditions.checkArgument; +import com.google.protobuf.ByteString; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; +import org.roaringbitmap.RoaringBitmap; + +@Slf4j +public class MutableBucket extends Bucket { + + private final TripleLongPriorityQueue priorityQueue; + + MutableBucket(ManagedCursor cursor, + BucketSnapshotStorage bucketSnapshotStorage, TripleLongPriorityQueue priorityQueue) { + super(cursor, bucketSnapshotStorage, -1L, -1L); + this.priorityQueue = priorityQueue; + } + + Pair sealBucketAndAsyncPersistent( + long timeStepPerBucketSnapshotSegment, + TripleLongPriorityQueue sharedQueue) { + if (priorityQueue.isEmpty()) { + return null; + } + long numMessages = 0; + + final long startLedgerId = getStartLedgerId(); + final long endLedgerId = getEndLedgerId(); + + List bucketSnapshotSegments = new ArrayList<>(); + List segmentMetadataList = new ArrayList<>(); + Map bitMap = new HashMap<>(); + SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); + SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); + + long currentTimestampUpperLimit = 0; + while (!priorityQueue.isEmpty()) { + long timestamp = priorityQueue.peekN1(); + if (currentTimestampUpperLimit == 0) { + currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1; + } + + long ledgerId = priorityQueue.peekN2(); + long entryId = priorityQueue.peekN3(); + + checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); + + // Move first segment of bucket snapshot to sharedBucketPriorityQueue + if (segmentMetadataList.size() == 0) { + sharedQueue.add(timestamp, ledgerId, entryId); + } + + priorityQueue.pop(); + numMessages++; + + DelayedIndex delayedIndex = DelayedIndex.newBuilder() + .setTimestamp(timestamp) + .setLedgerId(ledgerId) + .setEntryId(entryId).build(); + + bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); + + snapshotSegmentBuilder.addIndexes(delayedIndex); + + if (priorityQueue.isEmpty() || priorityQueue.peekN1() > currentTimestampUpperLimit) { + segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); + currentTimestampUpperLimit = 0; + + Iterator> iterator = bitMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + byte[] array = new byte[entry.getValue().serializedSizeInBytes()]; + entry.getValue().serialize(ByteBuffer.wrap(array)); + segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array)); + iterator.remove(); + } + + segmentMetadataList.add(segmentMetadataBuilder.build()); + segmentMetadataBuilder.clear(); + + bucketSnapshotSegments.add(snapshotSegmentBuilder.build()); + snapshotSegmentBuilder.clear(); + } + } + + SnapshotMetadata bucketSnapshotMetadata = SnapshotMetadata.newBuilder() + .addAllMetadataList(segmentMetadataList) + .build(); + + final int lastSegmentEntryId = segmentMetadataList.size(); + + ImmutableBucket bucket = new ImmutableBucket(cursor, bucketSnapshotStorage, startLedgerId, endLedgerId); + bucket.setCurrentSegmentEntryId(1); + bucket.setNumberBucketDelayedMessages(numMessages); + bucket.setLastSegmentEntryId(lastSegmentEntryId); + + // Add the first snapshot segment last message to snapshotSegmentLastMessageTable + checkArgument(!bucketSnapshotSegments.isEmpty()); + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); + DelayedIndex lastDelayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); + Pair result = Pair.of(bucket, lastDelayedIndex); + + CompletableFuture future = asyncSaveBucketSnapshot(bucket, + bucketSnapshotMetadata, bucketSnapshotSegments); + bucket.setSnapshotCreateFuture(future); + future.whenComplete((__, ex) -> { + if (ex == null) { + bucket.setSnapshotCreateFuture(null); + } else { + //TODO Record create snapshot failed + log.error("Failed to create snapshot: ", ex); + } + }); + + return result; + } + + void resetLastMutableBucketRange() { + this.setStartLedgerId(-1L); + this.setEndLedgerId(-1L); + } + + void clear() { + this.resetLastMutableBucketRange(); + this.delayedIndexBitMap.clear(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java new file mode 100644 index 0000000000000..0ab45ecded2cf --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java index 3a482ffb1ed56..c4fcc5a5ff9c5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java @@ -39,6 +39,8 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index 23b5c9b7e0817..89831a1d5e771 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; From 8405d137e3a5532cad98d15956a41130eee2a9aa Mon Sep 17 00:00:00 2001 From: coderzc Date: Fri, 4 Nov 2022 13:37:34 +0800 Subject: [PATCH 18/18] Abstract AbstractDelayedDeliveryTracker & remove `super.getPriorityQueue()` & rebase master --- .../AbstractDelayedDeliveryTracker.java | 162 +++++++++++ .../InMemoryDelayedDeliveryTracker.java | 137 +--------- .../pulsar/broker/delayed/bucket/Bucket.java | 2 +- .../bucket/BucketDelayedDeliveryTracker.java | 75 ++---- .../broker/delayed/bucket/MutableBucket.java | 54 +++- .../delayed/AbstractDeliveryTrackerTest.java | 240 +++++++++++++++++ .../BucketDelayedDeliveryTrackerTest.java | 30 +-- .../delayed/InMemoryDeliveryTrackerTest.java | 251 ++++-------------- 8 files changed, 528 insertions(+), 423 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java new file mode 100644 index 0000000000000..5c99e4c307d7c --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import java.time.Clock; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; + +@Slf4j +public abstract class AbstractDelayedDeliveryTracker implements DelayedDeliveryTracker, TimerTask { + + protected final PersistentDispatcherMultipleConsumers dispatcher; + + // Reference to the shared (per-broker) timer for delayed delivery + protected final Timer timer; + + // Current timeout or null if not set + protected Timeout timeout; + + // Timestamp at which the timeout is currently set + private long currentTimeoutTarget; + + // Last time the TimerTask was triggered for this class + private long lastTickRun; + + protected long tickTimeMillis; + + protected final Clock clock; + + private final boolean isDelayedDeliveryDeliverAtTimeStrict; + + public AbstractDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, + long tickTimeMillis, + boolean isDelayedDeliveryDeliverAtTimeStrict) { + this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); + } + + public AbstractDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, + long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict) { + this.dispatcher = dispatcher; + this.timer = timer; + this.tickTimeMillis = tickTimeMillis; + this.clock = clock; + this.isDelayedDeliveryDeliverAtTimeStrict = isDelayedDeliveryDeliverAtTimeStrict; + } + + + /** + * When {@link #isDelayedDeliveryDeliverAtTimeStrict} is false, we allow for early delivery by as much as the + * {@link #tickTimeMillis} because it is a slight optimization to let messages skip going back into the delay + * tracker for a brief amount of time when we're already trying to dispatch to the consumer. + * + * When {@link #isDelayedDeliveryDeliverAtTimeStrict} is true, we use the current time to determine when messages + * can be delivered. As a consequence, there are two delays that will affect delivery. The first is the + * {@link #tickTimeMillis} and the second is the {@link Timer}'s granularity. + * + * @return the cutoff time to determine whether a message is ready to deliver to the consumer + */ + protected long getCutoffTime() { + return isDelayedDeliveryDeliverAtTimeStrict ? clock.millis() : clock.millis() + tickTimeMillis; + } + + public void resetTickTime(long tickTime) { + if (this.tickTimeMillis != tickTime) { + this.tickTimeMillis = tickTime; + } + } + + protected void updateTimer() { + if (getNumberOfDelayedMessages() == 0) { + if (timeout != null) { + currentTimeoutTarget = -1; + timeout.cancel(); + timeout = null; + } + return; + } + long timestamp = nextDeliveryTime(); + if (timestamp == currentTimeoutTarget) { + // The timer is already set to the correct target time + return; + } + + if (timeout != null) { + timeout.cancel(); + } + + long now = clock.millis(); + long delayMillis = timestamp - now; + + if (delayMillis < 0) { + // There are messages that are already ready to be delivered. If + // the dispatcher is not getting them is because the consumer is + // either not connected or slow. + // We don't need to keep retriggering the timer. When the consumer + // catches up, the dispatcher will do the readMoreEntries() and + // get these messages + return; + } + + // Compute the earliest time that we schedule the timer to run. + long remainingTickDelayMillis = lastTickRun + tickTimeMillis - now; + long calculatedDelayMillis = Math.max(delayMillis, remainingTickDelayMillis); + + if (log.isDebugEnabled()) { + log.debug("[{}] Start timer in {} millis", dispatcher.getName(), calculatedDelayMillis); + } + + // Even though we may delay longer than this timestamp because of the tick delay, we still track the + // current timeout with reference to the next message's timestamp. + currentTimeoutTarget = timestamp; + timeout = timer.newTimeout(this, calculatedDelayMillis, TimeUnit.MILLISECONDS); + } + + @Override + public void run(Timeout timeout) throws Exception { + if (log.isDebugEnabled()) { + log.debug("[{}] Timer triggered", dispatcher.getName()); + } + if (timeout == null || timeout.isCancelled()) { + return; + } + + synchronized (dispatcher) { + lastTickRun = clock.millis(); + currentTimeoutTarget = -1; + this.timeout = null; + dispatcher.readMoreEntries(); + } + } + + @Override + public void close() { + if (timeout != null) { + timeout.cancel(); + timeout = null; + } + } + + protected abstract long nextDeliveryTime(); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 58c86deb410e1..f55d5fd11694b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -19,13 +19,10 @@ package org.apache.pulsar.broker.delayed; import com.google.common.annotations.VisibleForTesting; -import io.netty.util.Timeout; import io.netty.util.Timer; -import io.netty.util.TimerTask; import java.time.Clock; import java.util.NavigableSet; import java.util.TreeSet; -import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -33,30 +30,10 @@ import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @Slf4j -public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, TimerTask { +public class InMemoryDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker { protected final TripleLongPriorityQueue priorityQueue = new TripleLongPriorityQueue(); - protected final PersistentDispatcherMultipleConsumers dispatcher; - - // Reference to the shared (per-broker) timer for delayed delivery - private final Timer timer; - - // Current timeout or null if not set - protected Timeout timeout; - - // Timestamp at which the timeout is currently set - private long currentTimeoutTarget; - - // Last time the TimerTask was triggered for this class - private long lastTickRun; - - protected long tickTimeMillis; - - protected final Clock clock; - - private final boolean isDelayedDeliveryDeliverAtTimeStrict; - // If we detect that all messages have fixed delay time, such that the delivery is // always going to be in FIFO order, then we can avoid pulling all the messages in // tracker. Instead, we use the lookahead for detection and pause the read from @@ -84,29 +61,10 @@ public InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers disp long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict, long fixedDelayDetectionLookahead) { - this.dispatcher = dispatcher; - this.timer = timer; - this.tickTimeMillis = tickTimeMillis; - this.clock = clock; - this.isDelayedDeliveryDeliverAtTimeStrict = isDelayedDeliveryDeliverAtTimeStrict; + super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.fixedDelayDetectionLookahead = fixedDelayDetectionLookahead; } - /** - * When {@link #isDelayedDeliveryDeliverAtTimeStrict} is false, we allow for early delivery by as much as the - * {@link #tickTimeMillis} because it is a slight optimization to let messages skip going back into the delay - * tracker for a brief amount of time when we're already trying to dispatch to the consumer. - * - * When {@link #isDelayedDeliveryDeliverAtTimeStrict} is true, we use the current time to determine when messages - * can be delivered. As a consequence, there are two delays that will affect delivery. The first is the - * {@link #tickTimeMillis} and the second is the {@link Timer}'s granularity. - * - * @return the cutoff time to determine whether a message is ready to deliver to the consumer - */ - protected long getCutoffTime() { - return isDelayedDeliveryDeliverAtTimeStrict ? clock.millis() : clock.millis() + tickTimeMillis; - } - @Override public boolean addMessage(long ledgerId, long entryId, long deliverAt) { if (deliverAt < 0 || deliverAt <= getCutoffTime()) { @@ -119,7 +77,6 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { deliverAt - clock.millis()); } - priorityQueue.add(deliverAt, ledgerId, entryId); updateTimer(); @@ -189,14 +146,6 @@ public NavigableSet getScheduledMessages(int maxMessages) { return positions; } - @Override - public void resetTickTime(long tickTime) { - - if (this.tickTimeMillis != tickTime) { - this.tickTimeMillis = tickTime; - } - } - @Override public void clear() { this.priorityQueue.clear(); @@ -212,87 +161,10 @@ public long getBufferMemoryUsage() { return priorityQueue.bytesCapacity(); } - /** - * Update the scheduled timer task such that: - * 1. If there are no delayed messages, return and do not schedule a timer task. - * 2. If the next message in the queue has the same deliverAt time as the timer task, return and leave existing - * timer task in place. - * 3. If the deliverAt time for the next delayed message has already passed (i.e. the delay is negative), return - * without scheduling a timer task since the subscription is backlogged. - * 4. Else, schedule a timer task where the delay is the greater of these two: the next message's deliverAt time or - * the last tick time plus the tickTimeMillis (to ensure we do not schedule the task more frequently than the - * tickTimeMillis). - */ - protected void updateTimer() { - if (getNumberOfDelayedMessages() == 0) { - if (timeout != null) { - currentTimeoutTarget = -1; - timeout.cancel(); - timeout = null; - } - return; - } - long timestamp = nextDeliveryTime(); - if (timestamp == currentTimeoutTarget) { - // The timer is already set to the correct target time - return; - } - - if (timeout != null) { - timeout.cancel(); - } - - long now = clock.millis(); - long delayMillis = timestamp - now; - - if (delayMillis < 0) { - // There are messages that are already ready to be delivered. If - // the dispatcher is not getting them is because the consumer is - // either not connected or slow. - // We don't need to keep retriggering the timer. When the consumer - // catches up, the dispatcher will do the readMoreEntries() and - // get these messages - return; - } - - // Compute the earliest time that we schedule the timer to run. - long remainingTickDelayMillis = lastTickRun + tickTimeMillis - now; - long calculatedDelayMillis = Math.max(delayMillis, remainingTickDelayMillis); - - if (log.isDebugEnabled()) { - log.debug("[{}] Start timer in {} millis", dispatcher.getName(), calculatedDelayMillis); - } - - // Even though we may delay longer than this timestamp because of the tick delay, we still track the - // current timeout with reference to the next message's timestamp. - currentTimeoutTarget = timestamp; - timeout = timer.newTimeout(this, calculatedDelayMillis, TimeUnit.MILLISECONDS); - } - - @Override - public void run(Timeout timeout) throws Exception { - if (log.isDebugEnabled()) { - log.debug("[{}] Timer triggered", dispatcher.getName()); - } - if (timeout == null || timeout.isCancelled()) { - return; - } - - synchronized (dispatcher) { - lastTickRun = clock.millis(); - currentTimeoutTarget = -1; - this.timeout = null; - dispatcher.readMoreEntries(); - } - } - @Override public void close() { + super.close(); priorityQueue.close(); - if (timeout != null) { - timeout.cancel(); - timeout = null; - } } @Override @@ -309,9 +181,6 @@ public boolean containsMessage(long ledgerId, long entryId) { return false; } - protected TripleLongPriorityQueue getPriorityQueue() { - return priorityQueue; - } protected long nextDeliveryTime() { return priorityQueue.peekN1(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 25ff9d033e595..fbd6d765705d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -36,7 +36,7 @@ @Slf4j @Data @AllArgsConstructor -public abstract class Bucket { +abstract class Bucket { static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket"; static final String DELIMITER = "_"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 71b554fef41d5..b7f0e0a1bc1e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -28,8 +28,8 @@ import io.netty.util.Timer; import java.time.Clock; import java.util.Iterator; +import java.util.NavigableSet; import java.util.Optional; -import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -40,7 +40,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; @@ -48,7 +48,7 @@ @Slf4j @ThreadSafe -public class BucketDelayedDeliveryTracker extends InMemoryDelayedDeliveryTracker { +public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker { static final int AsyncOperationTimeoutSeconds = 30; @@ -84,35 +84,16 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat BucketSnapshotStorage bucketSnapshotStorage, long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, int maxNumBuckets) { - super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict, - -1L); + super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.minIndexCountPerBucket = minIndexCountPerBucket; this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; this.maxNumBuckets = maxNumBuckets; - ManagedCursor cursor = dispatcher.getCursor(); this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); this.immutableBuckets = TreeRangeMap.create(); this.snapshotSegmentLastIndexTable = HashBasedTable.create(); - this.numberDelayedMessages = 0L; - - this.lastMutableBucket = new MutableBucket(cursor, bucketSnapshotStorage, super.getPriorityQueue()); - } - - private void moveScheduledMessageToSharedQueue(long cutoffTime) { - TripleLongPriorityQueue priorityQueue = getPriorityQueue(); - while (!priorityQueue.isEmpty()) { - long timestamp = priorityQueue.peekN1(); - if (timestamp > cutoffTime) { - break; - } - - long ledgerId = priorityQueue.peekN2(); - long entryId = priorityQueue.peekN3(); - sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); - - priorityQueue.pop(); - } + ManagedCursor cursor = dispatcher.getCursor(); + this.lastMutableBucket = new MutableBucket(cursor, bucketSnapshotStorage); } @Override @@ -121,7 +102,7 @@ public void run(Timeout timeout) throws Exception { if (timeout == null || timeout.isCancelled()) { return; } - moveScheduledMessageToSharedQueue(getCutoffTime()); + lastMutableBucket.moveScheduledMessageToSharedQueue(getCutoffTime(), sharedBucketPriorityQueue); } super.run(timeout); } @@ -166,13 +147,14 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver boolean existBucket = findImmutableBucket(ledgerId).isPresent(); // Create bucket snapshot - if (ledgerId > lastMutableBucket.endLedgerId && !getPriorityQueue().isEmpty()) { - if (getPriorityQueue().size() >= minIndexCountPerBucket && !existBucket) { - sealBucket(); - lastMutableBucket.resetLastMutableBucketRange(); - if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { - // TODO merge bucket snapshot (synchronize operate) - } + if (!existBucket && ledgerId > lastMutableBucket.endLedgerId + && lastMutableBucket.size() >= minIndexCountPerBucket + && !lastMutableBucket.isEmpty()) { + sealBucket(); + lastMutableBucket.resetLastMutableBucketRange(); + + if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { + // TODO merge bucket snapshot (synchronize operate) } } @@ -182,16 +164,9 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); } else { checkArgument(ledgerId >= lastMutableBucket.endLedgerId); - - getPriorityQueue().add(deliverAt, ledgerId, entryId); - - if (lastMutableBucket.startLedgerId == -1L) { - lastMutableBucket.setStartLedgerId(ledgerId); - } - lastMutableBucket.setEndLedgerId(ledgerId); + lastMutableBucket.addMessage(ledgerId, entryId, deliverAt); } - lastMutableBucket.putIndexBit(ledgerId, entryId); numberDelayedMessages++; if (log.isDebugEnabled()) { @@ -217,12 +192,12 @@ public synchronized boolean hasMessageAvailable() { @Override protected long nextDeliveryTime() { - if (getPriorityQueue().isEmpty() && !sharedBucketPriorityQueue.isEmpty()) { + if (lastMutableBucket.isEmpty() && !sharedBucketPriorityQueue.isEmpty()) { return sharedBucketPriorityQueue.peekN1(); - } else if (sharedBucketPriorityQueue.isEmpty() && !getPriorityQueue().isEmpty()) { - return getPriorityQueue().peekN1(); + } else if (sharedBucketPriorityQueue.isEmpty() && !lastMutableBucket.isEmpty()) { + return lastMutableBucket.nextDeliveryTime(); } - long timestamp = getPriorityQueue().peekN1(); + long timestamp = lastMutableBucket.nextDeliveryTime(); long bucketTimestamp = sharedBucketPriorityQueue.peekN1(); return Math.min(timestamp, bucketTimestamp); } @@ -234,16 +209,16 @@ public synchronized long getNumberOfDelayedMessages() { @Override public synchronized long getBufferMemoryUsage() { - return getPriorityQueue().bytesCapacity() + sharedBucketPriorityQueue.bytesCapacity(); + return this.lastMutableBucket.getBufferMemoryUsage() + sharedBucketPriorityQueue.bytesCapacity(); } @Override - public synchronized Set getScheduledMessages(int maxMessages) { + public synchronized NavigableSet getScheduledMessages(int maxMessages) { long cutoffTime = getCutoffTime(); - moveScheduledMessageToSharedQueue(cutoffTime); + lastMutableBucket.moveScheduledMessageToSharedQueue(cutoffTime, sharedBucketPriorityQueue); - Set positions = new TreeSet<>(); + NavigableSet positions = new TreeSet<>(); int n = maxMessages; while (n > 0 && !sharedBucketPriorityQueue.isEmpty()) { @@ -302,7 +277,6 @@ public boolean shouldPauseAllDeliveries() { @Override public synchronized void clear() { - super.clear(); cleanImmutableBuckets(true); sharedBucketPriorityQueue.clear(); lastMutableBucket.clear(); @@ -313,6 +287,7 @@ public synchronized void clear() { @Override public synchronized void close() { super.close(); + lastMutableBucket.close(); cleanImmutableBuckets(false); sharedBucketPriorityQueue.close(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index 4673809105b04..36026298269d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -38,14 +38,14 @@ import org.roaringbitmap.RoaringBitmap; @Slf4j -public class MutableBucket extends Bucket { +class MutableBucket extends Bucket implements AutoCloseable { private final TripleLongPriorityQueue priorityQueue; MutableBucket(ManagedCursor cursor, - BucketSnapshotStorage bucketSnapshotStorage, TripleLongPriorityQueue priorityQueue) { + BucketSnapshotStorage bucketSnapshotStorage) { super(cursor, bucketSnapshotStorage, -1L, -1L); - this.priorityQueue = priorityQueue; + this.priorityQueue = new TripleLongPriorityQueue(); } Pair sealBucketAndAsyncPersistent( @@ -147,13 +147,57 @@ Pair sealBucketAndAsyncPersistent( return result; } + void moveScheduledMessageToSharedQueue(long cutoffTime, TripleLongPriorityQueue sharedBucketPriorityQueue) { + while (!priorityQueue.isEmpty()) { + long timestamp = priorityQueue.peekN1(); + if (timestamp > cutoffTime) { + break; + } + + long ledgerId = priorityQueue.peekN2(); + long entryId = priorityQueue.peekN3(); + sharedBucketPriorityQueue.add(timestamp, ledgerId, entryId); + + priorityQueue.pop(); + } + } + void resetLastMutableBucketRange() { - this.setStartLedgerId(-1L); - this.setEndLedgerId(-1L); + this.startLedgerId = -1L; + this.endLedgerId = -1L; } void clear() { this.resetLastMutableBucketRange(); this.delayedIndexBitMap.clear(); } + + public void close() { + priorityQueue.close(); + } + + long getBufferMemoryUsage() { + return priorityQueue.bytesCapacity(); + } + + boolean isEmpty() { + return priorityQueue.isEmpty(); + } + + long nextDeliveryTime() { + return priorityQueue.peekN1(); + } + + long size() { + return priorityQueue.size(); + } + + void addMessage(long ledgerId, long entryId, long deliverAt) { + priorityQueue.add(deliverAt, ledgerId, entryId); + if (startLedgerId == -1L) { + this.startLedgerId = ledgerId; + } + this.endLedgerId = ledgerId; + putIndexBit(ledgerId, entryId); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java new file mode 100644 index 0000000000000..1d166a8db5c9e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import static org.mockito.Mockito.atMostOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.time.Clock; +import java.util.Collections; +import java.util.NavigableMap; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +public abstract class AbstractDeliveryTrackerTest { + + // Create a single shared timer for the test. + protected final Timer timer = + new HashedWheelTimer(new DefaultThreadFactory("pulsar-in-memory-delayed-delivery-test"), + 500, TimeUnit.MILLISECONDS); + protected PersistentDispatcherMultipleConsumers dispatcher; + protected Clock clock; + + protected AtomicLong clockTime; + + @AfterClass(alwaysRun = true) + public void cleanup() { + timer.stop(); + } + + @Test(dataProvider = "delayedTracker") + public void test(DelayedDeliveryTracker tracker) throws Exception { + assertFalse(tracker.hasMessageAvailable()); + + assertTrue(tracker.addMessage(1, 2, 20)); + assertTrue(tracker.addMessage(2, 1, 10)); + assertTrue(tracker.addMessage(3, 3, 30)); + assertTrue(tracker.addMessage(4, 5, 50)); + assertTrue(tracker.addMessage(5, 4, 40)); + + assertFalse(tracker.hasMessageAvailable()); + assertEquals(tracker.getNumberOfDelayedMessages(), 5); + + assertEquals(tracker.getScheduledMessages(10), Collections.emptySet()); + + // Move time forward + clockTime.set(15); + + // Message is rejected by tracker since it's already ready to send + assertFalse(tracker.addMessage(6, 6, 10)); + + assertEquals(tracker.getNumberOfDelayedMessages(), 5); + assertTrue(tracker.hasMessageAvailable()); + Set scheduled = tracker.getScheduledMessages(10); + assertEquals(scheduled.size(), 1); + + // Move time forward + clockTime.set(60); + + assertEquals(tracker.getNumberOfDelayedMessages(), 4); + assertTrue(tracker.hasMessageAvailable()); + scheduled = tracker.getScheduledMessages(1); + assertEquals(scheduled.size(), 1); + + assertEquals(tracker.getNumberOfDelayedMessages(), 3); + assertTrue(tracker.hasMessageAvailable()); + scheduled = tracker.getScheduledMessages(3); + assertEquals(scheduled.size(), 3); + + assertEquals(tracker.getNumberOfDelayedMessages(), 0); + assertFalse(tracker.hasMessageAvailable()); + assertEquals(tracker.getScheduledMessages(10), Collections.emptySet()); + + tracker.close(); + } + + @Test(dataProvider = "delayedTracker") + public void testWithTimer(DelayedDeliveryTracker tracker, NavigableMap tasks) throws Exception { + assertTrue(tasks.isEmpty()); + assertTrue(tracker.addMessage(2, 2, 20)); + assertEquals(tasks.size(), 1); + assertEquals(tasks.firstKey().longValue(), 20); + + assertTrue(tracker.addMessage(1, 1, 10)); + assertEquals(tasks.size(), 1); + assertEquals(tasks.firstKey().longValue(), 10); + + assertTrue(tracker.addMessage(3, 3, 30)); + assertEquals(tasks.size(), 1); + assertEquals(tasks.firstKey().longValue(), 10); + + clockTime.set(15); + + TimerTask task = tasks.pollFirstEntry().getValue(); + Timeout cancelledTimeout = mock(Timeout.class); + when(cancelledTimeout.isCancelled()).thenReturn(true); + task.run(cancelledTimeout); + verify(dispatcher, atMostOnce()).readMoreEntries(); + + task.run(mock(Timeout.class)); + verify(dispatcher).readMoreEntries(); + + tracker.close(); + } + + /** + * Adding a message that is about to expire within the tick time should lead + * to a rejection from the tracker when isDelayedDeliveryDeliverAtTimeStrict is false. + */ + @Test(dataProvider = "delayedTracker") + public void testAddWithinTickTime(DelayedDeliveryTracker tracker) { + clockTime.set(0); + + assertFalse(tracker.addMessage(1, 1, 10)); + assertFalse(tracker.addMessage(2, 2, 99)); + assertFalse(tracker.addMessage(3, 3, 100)); + assertTrue(tracker.addMessage(4, 4, 101)); + assertTrue(tracker.addMessage(5, 5, 200)); + + assertEquals(tracker.getNumberOfDelayedMessages(), 2); + + tracker.close(); + } + + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithStrictDelay(DelayedDeliveryTracker tracker) { + clockTime.set(10); + + // Verify behavior for the less than, equal to, and greater than deliverAt times. + assertFalse(tracker.addMessage(1, 1, 9)); + assertFalse(tracker.addMessage(4, 4, 10)); + assertTrue(tracker.addMessage(1, 1, 11)); + + assertEquals(tracker.getNumberOfDelayedMessages(), 1); + assertFalse(tracker.hasMessageAvailable()); + + tracker.close(); + } + + /** + * In this test, the deliverAt time is after now, but the deliverAt time is too early to run another tick, so the + * tickTimeMillis determines the delay. + */ + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict(DelayedDeliveryTracker tracker) + throws Exception { + // Set clock time, then run tracker to inherit clock time as the last tick time. + clockTime.set(10000); + Timeout timeout = mock(Timeout.class); + when(timeout.isCancelled()).then(x -> false); + ((AbstractDelayedDeliveryTracker) tracker).run(timeout); + verify(dispatcher, times(1)).readMoreEntries(); + + // Add a message that has a delivery time just after the previous run. It will get delivered based on the + // tick delay plus the last tick run. + assertTrue(tracker.addMessage(1, 1, 10001)); + + // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has + // passed where it would have been triggered if the tick time was doing the triggering. + Thread.sleep(600); + verify(dispatcher, times(1)).readMoreEntries(); + + // Not wait for the message delivery to get triggered. + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + + tracker.close(); + } + + /** + * In this test, the deliverAt time is after now, but before the (tickTimeMillis + now). Because there wasn't a + * recent tick run, the deliverAt time determines the delay. + */ + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict( + DelayedDeliveryTracker tracker) { + clockTime.set(500000); + + assertTrue(tracker.addMessage(1, 1, 500005)); + + // Wait long enough for the runnable to run, but not longer than the tick time. The point is that the delivery + // should get scheduled early when the tick duration has passed since the last tick. + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + + tracker.close(); + } + + /** + * In this test, the deliverAt time is after now plus tickTimeMillis, so the tickTimeMillis determines the delay. + */ + @Test(dataProvider = "delayedTracker") + public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict(DelayedDeliveryTracker tracker) + throws Exception { + clockTime.set(0); + + assertTrue(tracker.addMessage(1, 1, 2000)); + + // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has + // passed where it would have been triggered if the tick time was doing the triggering. + Thread.sleep(1000); + + // Not wait for the message delivery to get triggered. + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + + tracker.close(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java index c4fcc5a5ff9c5..331ceb83a9994 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerTest.java @@ -25,11 +25,9 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; -import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Method; import java.time.Clock; import java.util.NavigableMap; @@ -46,10 +44,8 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class BucketDelayedDeliveryTrackerTest extends InMemoryDeliveryTrackerTest { - - private final Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-bucket-delayed-delivery-test"), - 500, TimeUnit.MILLISECONDS); +@Test(groups = "broker") +public class BucketDelayedDeliveryTrackerTest extends AbstractDeliveryTrackerTest { private BucketSnapshotStorage bucketSnapshotStorage; @@ -61,7 +57,6 @@ public void clean() throws Exception { } @DataProvider(name = "delayedTracker") - @Override public Object[][] provider(Method method) throws Exception { dispatcher = mock(PersistentDispatcherMultipleConsumers.class); clock = mock(Clock.class); @@ -161,25 +156,4 @@ public void testContainsMessage(DelayedDeliveryTracker tracker) { tracker.close(); } - - @Override - @Test(dataProvider = "delayedTracker") - public void testWithFixedDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { - assertEquals(tracker.getFixedDelayDetectionLookahead(), -1L); - tracker.close(); - } - - @Override - @Test(dataProvider = "delayedTracker") - public void testWithMixedDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { - assertEquals(tracker.getFixedDelayDetectionLookahead(), -1L); - tracker.close(); - } - - @Override - @Test(dataProvider = "delayedTracker") - public void testWithNoDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { - assertEquals(tracker.getFixedDelayDetectionLookahead(), -1L); - tracker.close(); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 7d50c2a05cdc6..6711aed924c20 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,17 +18,14 @@ */ package org.apache.pulsar.broker.delayed; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyLong; -import static org.mockito.Mockito.atMostOnce; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.Timer; @@ -36,34 +33,16 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Method; import java.time.Clock; -import java.util.Collections; import java.util.NavigableMap; -import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; -import org.awaitility.Awaitility; -import org.testng.annotations.AfterClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker") -public class InMemoryDeliveryTrackerTest { - - // Create a single shared timer for the test. - private final Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-in-memory-delayed-delivery-test"), - 500, TimeUnit.MILLISECONDS); - protected PersistentDispatcherMultipleConsumers dispatcher; - protected Clock clock; - - protected AtomicLong clockTime; - - @AfterClass(alwaysRun = true) - public void cleanup() { - timer.stop(); - } +public class InMemoryDeliveryTrackerTest extends AbstractDeliveryTrackerTest { @DataProvider(name = "delayedTracker") public Object[][] provider(Method method) throws Exception { @@ -139,185 +118,6 @@ public Object[][] provider(Method method) throws Exception { }; } - @Test(dataProvider = "delayedTracker") - public void test(DelayedDeliveryTracker tracker) throws Exception { - assertFalse(tracker.hasMessageAvailable()); - - assertTrue(tracker.addMessage(1, 2, 20)); - assertTrue(tracker.addMessage(2, 1, 10)); - assertTrue(tracker.addMessage(3, 3, 30)); - assertTrue(tracker.addMessage(4, 5, 50)); - assertTrue(tracker.addMessage(5, 4, 40)); - - assertFalse(tracker.hasMessageAvailable()); - assertEquals(tracker.getNumberOfDelayedMessages(), 5); - - assertEquals(tracker.getScheduledMessages(10), Collections.emptySet()); - - // Move time forward - clockTime.set(15); - - // Message is rejected by tracker since it's already ready to send - assertFalse(tracker.addMessage(6, 6, 10)); - - assertEquals(tracker.getNumberOfDelayedMessages(), 5); - assertTrue(tracker.hasMessageAvailable()); - Set scheduled = tracker.getScheduledMessages(10); - assertEquals(scheduled.size(), 1); - - // Move time forward - clockTime.set(60); - - assertEquals(tracker.getNumberOfDelayedMessages(), 4); - assertTrue(tracker.hasMessageAvailable()); - scheduled = tracker.getScheduledMessages(1); - assertEquals(scheduled.size(), 1); - - assertEquals(tracker.getNumberOfDelayedMessages(), 3); - assertTrue(tracker.hasMessageAvailable()); - scheduled = tracker.getScheduledMessages(3); - assertEquals(scheduled.size(), 3); - - assertEquals(tracker.getNumberOfDelayedMessages(), 0); - assertFalse(tracker.hasMessageAvailable()); - assertEquals(tracker.getScheduledMessages(10), Collections.emptySet()); - - tracker.close(); - } - - @Test(dataProvider = "delayedTracker") - public void testWithTimer(DelayedDeliveryTracker tracker, NavigableMap tasks) throws Exception { - assertTrue(tasks.isEmpty()); - assertTrue(tracker.addMessage(2, 2, 20)); - assertEquals(tasks.size(), 1); - assertEquals(tasks.firstKey().longValue(), 20); - - assertTrue(tracker.addMessage(1, 1, 10)); - assertEquals(tasks.size(), 1); - assertEquals(tasks.firstKey().longValue(), 10); - - assertTrue(tracker.addMessage(3, 3, 30)); - assertEquals(tasks.size(), 1); - assertEquals(tasks.firstKey().longValue(), 10); - - clockTime.set(15); - - TimerTask task = tasks.pollFirstEntry().getValue(); - Timeout cancelledTimeout = mock(Timeout.class); - when(cancelledTimeout.isCancelled()).thenReturn(true); - task.run(cancelledTimeout); - verify(dispatcher, atMostOnce()).readMoreEntries(); - - task.run(mock(Timeout.class)); - verify(dispatcher).readMoreEntries(); - - tracker.close(); - } - - /** - * Adding a message that is about to expire within the tick time should lead - * to a rejection from the tracker when isDelayedDeliveryDeliverAtTimeStrict is false. - */ - @Test(dataProvider = "delayedTracker") - public void testAddWithinTickTime(DelayedDeliveryTracker tracker) { - clockTime.set(0); - - assertFalse(tracker.addMessage(1, 1, 10)); - assertFalse(tracker.addMessage(2, 2, 99)); - assertFalse(tracker.addMessage(3, 3, 100)); - assertTrue(tracker.addMessage(4, 4, 101)); - assertTrue(tracker.addMessage(5, 5, 200)); - - assertEquals(tracker.getNumberOfDelayedMessages(), 2); - - tracker.close(); - } - - @Test(dataProvider = "delayedTracker") - public void testAddMessageWithStrictDelay(DelayedDeliveryTracker tracker) { - clockTime.set(10); - - // Verify behavior for the less than, equal to, and greater than deliverAt times. - assertFalse(tracker.addMessage(1, 1, 9)); - assertFalse(tracker.addMessage(4, 4, 10)); - assertTrue(tracker.addMessage(1, 1, 11)); - - assertEquals(tracker.getNumberOfDelayedMessages(), 1); - assertFalse(tracker.hasMessageAvailable()); - - tracker.close(); - } - - /** - * In this test, the deliverAt time is after now, but the deliverAt time is too early to run another tick, so the - * tickTimeMillis determines the delay. - */ - @Test(dataProvider = "delayedTracker") - public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict(DelayedDeliveryTracker tracker) - throws Exception { - // Set clock time, then run tracker to inherit clock time as the last tick time. - clockTime.set(10000); - Timeout timeout = mock(Timeout.class); - when(timeout.isCancelled()).then(x -> false); - ((InMemoryDelayedDeliveryTracker) tracker).run(timeout); - verify(dispatcher, times(1)).readMoreEntries(); - - // Add a message that has a delivery time just after the previous run. It will get delivered based on the - // tick delay plus the last tick run. - assertTrue(tracker.addMessage(1, 1, 10001)); - - // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has - // passed where it would have been triggered if the tick time was doing the triggering. - Thread.sleep(600); - verify(dispatcher, times(1)).readMoreEntries(); - - // Not wait for the message delivery to get triggered. - Awaitility.await().atMost(10, TimeUnit.SECONDS) - .untilAsserted(() -> verify(dispatcher).readMoreEntries()); - - tracker.close(); - } - - /** - * In this test, the deliverAt time is after now, but before the (tickTimeMillis + now). Because there wasn't a - * recent tick run, the deliverAt time determines the delay. - */ - @Test(dataProvider = "delayedTracker") - public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict( - DelayedDeliveryTracker tracker) { - clockTime.set(500000); - - assertTrue(tracker.addMessage(1, 1, 500005)); - - // Wait long enough for the runnable to run, but not longer than the tick time. The point is that the delivery - // should get scheduled early when the tick duration has passed since the last tick. - Awaitility.await().atMost(10, TimeUnit.SECONDS) - .untilAsserted(() -> verify(dispatcher).readMoreEntries()); - - tracker.close(); - } - - /** - * In this test, the deliverAt time is after now plus tickTimeMillis, so the tickTimeMillis determines the delay. - */ - @Test(dataProvider = "delayedTracker") - public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict(DelayedDeliveryTracker tracker) - throws Exception { - clockTime.set(0); - - assertTrue(tracker.addMessage(1, 1, 2000)); - - // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has - // passed where it would have been triggered if the tick time was doing the triggering. - Thread.sleep(1000); - - // Not wait for the message delivery to get triggered. - Awaitility.await().atMost(10, TimeUnit.SECONDS) - .untilAsserted(() -> verify(dispatcher).readMoreEntries()); - - tracker.close(); - } - @Test(dataProvider = "delayedTracker") public void testWithFixedDelays(InMemoryDelayedDeliveryTracker tracker) throws Exception { assertFalse(tracker.hasMessageAvailable()); @@ -407,4 +207,45 @@ public void testWithNoDelays(InMemoryDelayedDeliveryTracker tracker) throws Exce tracker.close(); } + @Test + public void testClose() throws Exception { + Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-in-memory-delayed-delivery-test"), + 1, TimeUnit.MILLISECONDS); + + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + final Exception[] exceptions = new Exception[1]; + + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true, 0) { + @Override + public void run(Timeout timeout) throws Exception { + super.timeout = timer.newTimeout(this, 1, TimeUnit.MILLISECONDS); + if (timeout == null || timeout.isCancelled()) { + return; + } + try { + this.priorityQueue.peekN1(); + } catch (Exception e) { + e.printStackTrace(); + exceptions[0] = e; + } + } + }; + + tracker.addMessage(1, 1, 10); + clockTime.set(10); + + Thread.sleep(300); + + tracker.close(); + + assertNull(exceptions[0]); + + timer.stop(); + } }