From 9d4327cbadb408c8134e5ac09592d41719f772e5 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Wed, 5 Aug 2020 18:25:59 +0100 Subject: [PATCH 01/14] key/value reverse operation --- .../internals/AbstractReadOnlyDecorator.java | 26 +++++++++++++++++++ .../state/internals/KeyValueSegmentsTest.java | 20 ++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java index 74227af3fa20e..90f5a23efd1a6 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java @@ -252,6 +252,13 @@ public KeyValueIterator, AGG> findSessions(final K key, return wrapped().findSessions(key, earliestSessionEndTime, latestSessionStartTime); } + @Override + public KeyValueIterator, AGG> backwardFindSessions(final K key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + return wrapped().backwardFindSessions(key, earliestSessionEndTime, latestSessionStartTime); + } + @Override public KeyValueIterator, AGG> findSessions(final K keyFrom, final K keyTo, @@ -260,6 +267,14 @@ public KeyValueIterator, AGG> findSessions(final K keyFrom, return wrapped().findSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); } + @Override + public KeyValueIterator, AGG> backwardFindSessions(final K keyFrom, + final K keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + return wrapped().backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + } + @Override public void remove(final Windowed sessionKey) { throw new UnsupportedOperationException(ERROR_MESSAGE); @@ -281,10 +296,21 @@ public KeyValueIterator, AGG> fetch(final K key) { return wrapped().fetch(key); } + @Override + public KeyValueIterator, AGG> backwardFetch(final K key) { + return wrapped().backwardFetch(key); + } + @Override public KeyValueIterator, AGG> fetch(final K from, final K to) { return wrapped().fetch(from, to); } + + @Override + public KeyValueIterator, AGG> backwardFetch(final K from, + final K to) { + return wrapped().backwardFetch(from, to); + } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java index aeef8ce8e13f5..d88cbd0011866 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java @@ -187,7 +187,11 @@ public void shouldGetSegmentsWithinTimeRange() { segments.getOrCreateSegmentIfLive(3, context, streamTime); segments.getOrCreateSegmentIfLive(4, context, streamTime); +<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); +======= + final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); +>>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(0, segments.get(0).id); assertEquals(1, segments.get(1).id); @@ -207,7 +211,11 @@ public void shouldGetSegmentsWithinBackwardTimeRange() { segments.getOrCreateSegmentIfLive(3, context, streamTime); segments.getOrCreateSegmentIfLive(4, context, streamTime); +<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); +======= + final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); +>>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(0, segments.get(2).id); assertEquals(1, segments.get(1).id); @@ -222,7 +230,11 @@ public void shouldGetSegmentsWithinTimeRangeOutOfOrder() { updateStreamTimeAndCreateSegment(1); updateStreamTimeAndCreateSegment(3); +<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); +======= + final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); +>>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(0, segments.get(0).id); assertEquals(1, segments.get(1).id); @@ -237,7 +249,11 @@ public void shouldGetSegmentsWithinTimeBackwardRangeOutOfOrder() { updateStreamTimeAndCreateSegment(1); updateStreamTimeAndCreateSegment(3); +<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); +======= + final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); +>>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(2, segments.get(0).id); assertEquals(1, segments.get(1).id); @@ -344,7 +360,11 @@ public void shouldClearSegmentsOnClose() { } private void verifyCorrectSegments(final long first, final int numSegments) { +<<<<<<< HEAD final List result = this.segments.segments(0, Long.MAX_VALUE, true); +======= + final List result = this.segments.segments(0, Long.MAX_VALUE, false); +>>>>>>> e431f38ea... key/value reverse operation assertEquals(numSegments, result.size()); for (int i = 0; i < numSegments; i++) { assertEquals(i + first, result.get(i).id); From 6761c71e41040883634c8b664cdac9b3250a778c Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Wed, 5 Aug 2020 18:33:56 +0100 Subject: [PATCH 02/14] window backward operations --- .../internals/AbstractReadOnlyDecorator.java | 26 ----------------- .../ChangeLoggingWindowBytesStore.java | 10 +++++++ .../internals/ReadOnlyWindowStoreFacade.java | 28 +++++++++++++++++++ .../ChangeLoggingWindowBytesStoreTest.java | 2 ++ 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java index 90f5a23efd1a6..74227af3fa20e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractReadOnlyDecorator.java @@ -252,13 +252,6 @@ public KeyValueIterator, AGG> findSessions(final K key, return wrapped().findSessions(key, earliestSessionEndTime, latestSessionStartTime); } - @Override - public KeyValueIterator, AGG> backwardFindSessions(final K key, - final long earliestSessionEndTime, - final long latestSessionStartTime) { - return wrapped().backwardFindSessions(key, earliestSessionEndTime, latestSessionStartTime); - } - @Override public KeyValueIterator, AGG> findSessions(final K keyFrom, final K keyTo, @@ -267,14 +260,6 @@ public KeyValueIterator, AGG> findSessions(final K keyFrom, return wrapped().findSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); } - @Override - public KeyValueIterator, AGG> backwardFindSessions(final K keyFrom, - final K keyTo, - final long earliestSessionEndTime, - final long latestSessionStartTime) { - return wrapped().backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); - } - @Override public void remove(final Windowed sessionKey) { throw new UnsupportedOperationException(ERROR_MESSAGE); @@ -296,21 +281,10 @@ public KeyValueIterator, AGG> fetch(final K key) { return wrapped().fetch(key); } - @Override - public KeyValueIterator, AGG> backwardFetch(final K key) { - return wrapped().backwardFetch(key); - } - @Override public KeyValueIterator, AGG> fetch(final K from, final K to) { return wrapped().fetch(from, to); } - - @Override - public KeyValueIterator, AGG> backwardFetch(final K from, - final K to) { - return wrapped().backwardFetch(from, to); - } } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java index 8da413cab24bb..a7635b304eea8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java @@ -25,6 +25,8 @@ import org.apache.kafka.streams.state.WindowStore; import org.apache.kafka.streams.state.WindowStoreIterator; +import java.time.Instant; + /** * Simple wrapper around a {@link WindowStore} to support writing * updates to a changelog @@ -93,6 +95,14 @@ public KeyValueIterator, byte[]> backwardFetch(final Bytes keyFr return wrapped().backwardFetch(keyFrom, keyTo, timeFrom, timeTo); } + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes keyFrom, + final Bytes keyTo, + final Instant from, + final Instant to) { + return wrapped().backwardFetch(keyFrom, keyTo, from, to); + } + @Override public KeyValueIterator, byte[]> all() { return wrapped().all(); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java index b9fdebeca7c20..ecbcb5a841f8e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java @@ -63,6 +63,13 @@ public WindowStoreIterator backwardFetch(final K key, return new WindowStoreIteratorFacade<>(inner.backwardFetch(key, timeFrom, timeTo)); } + @Override + public WindowStoreIterator backwardFetch(final K key, + final Instant from, + final Instant to) throws IllegalArgumentException { + return new WindowStoreIteratorFacade<>(inner.backwardFetch(key, from, to)); + } + @Override @SuppressWarnings("deprecation") public KeyValueIterator, V> fetch(final K keyFrom, @@ -88,6 +95,14 @@ public KeyValueIterator, V> backwardFetch(final K keyFrom, return new KeyValueIteratorFacade<>(inner.backwardFetch(keyFrom, keyTo, timeFrom, timeTo)); } + @Override + public KeyValueIterator, V> backwardFetch(final K from, + final K to, + final Instant fromTime, + final Instant toTime) throws IllegalArgumentException { + return new KeyValueIteratorFacade<>(inner.backwardFetch(from, to, fromTime, toTime)); + } + @Override @SuppressWarnings("deprecation") public KeyValueIterator, V> fetchAll(final long timeFrom, @@ -107,6 +122,13 @@ public KeyValueIterator, V> backwardFetchAll(final Instant timeFrom, return new KeyValueIteratorFacade<>(inner.backwardFetchAll(timeFrom, timeTo)); } + @Override + public KeyValueIterator, V> backwardFetchAll(final Instant from, + final Instant to) throws IllegalArgumentException { + final KeyValueIterator, ValueAndTimestamp> innerIterator = inner.backwardFetchAll(from, to); + return new KeyValueIteratorFacade<>(innerIterator); + } + @Override public KeyValueIterator, V> all() { return new KeyValueIteratorFacade<>(inner.all()); @@ -117,6 +139,12 @@ public KeyValueIterator, V> backwardAll() { return new KeyValueIteratorFacade<>(inner.backwardAll()); } + @Override + public KeyValueIterator, V> backwardAll() { + final KeyValueIterator, ValueAndTimestamp> innerIterator = inner.backwardAll(); + return new KeyValueIteratorFacade<>(innerIterator); + } + private static class WindowStoreIteratorFacade implements WindowStoreIterator { final KeyValueIterator> innerIterator; diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java index c877ac64f5ab1..c4981ab0f40be 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java @@ -32,6 +32,8 @@ import org.junit.Test; import org.junit.runner.RunWith; +import java.time.Instant; + import static java.time.Instant.ofEpochMilli; @RunWith(EasyMockRunner.class) From 000c7b79009d24f61a97e562d8cdf5fb2bbef637 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Wed, 5 Aug 2020 18:35:24 +0100 Subject: [PATCH 03/14] session backward operations --- .../kafka/connect/runtime/SessionKey.java | 3 +- .../streams/state/ReadOnlySessionStore.java | 143 ++++++++++++- .../kafka/streams/state/SessionStore.java | 42 +--- .../state/internals/CachingSessionStore.java | 93 +++++++-- .../ChangeLoggingSessionBytesStore.java | 20 ++ .../CompositeReadOnlySessionStore.java | 162 ++++++++++++++- .../state/internals/InMemorySessionStore.java | 147 +++++++++++-- ...MergedSortedCacheSessionStoreIterator.java | 9 +- .../state/internals/MeteredSessionStore.java | 62 ++++++ .../internals/ReadOnlyWindowStoreFacade.java | 28 --- .../state/internals/RocksDBSessionStore.java | 37 +++- .../state/internals/SessionKeySchema.java | 1 - .../AbstractSessionBytesStoreTest.java | 195 +++++++++++++++++- .../internals/CachingSessionStoreTest.java | 123 ++++++++--- .../ChangeLoggingSessionBytesStoreTest.java | 42 +++- .../state/internals/KeyValueSegmentsTest.java | 20 -- ...dCacheWrappedSessionStoreIteratorTest.java | 65 +++++- .../internals/MeteredSessionStoreTest.java | 95 +++++++-- .../kafka/test/ReadOnlySessionStoreStub.java | 46 ++++- 19 files changed, 1127 insertions(+), 206 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java index ab5476e4b9000..2ff0cc84c5ac1 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java @@ -29,7 +29,8 @@ public class SessionKey { /** * Create a new session key with the given key value and creation timestamp - * @param key the actual cryptographic key to use for request validation; may not be null + * + * @param key the actual cryptographic key to use for request validation; may not be null * @param creationTimestamp the time at which the key was generated */ public SessionKey(SecretKey key, long creationTimestamp) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java index 230d2576178da..018dbedb3cabc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java @@ -24,35 +24,156 @@ * Implementations should be thread-safe as concurrent reads and writes * are expected. * - * @param the key type + * @param the key type * @param the aggregated value type */ public interface ReadOnlySessionStore { + /** - * Retrieve all aggregated sessions for the provided key. + * Fetch any sessions with the matching key and the sessions end is ≥ earliestSessionEndTime and the sessions + * start is ≤ latestSessionStartTime iterating from earliest to latest. + *

* This iterator must be closed after use. * + * @param key the key to return sessions for + * @param earliestSessionEndTime the end timestamp of the earliest session to search for + * @param latestSessionStartTime the end timestamp of the latest session to search for + * @return iterator of sessions with the matching key and aggregated values + * @throws NullPointerException If null is used for key. + */ + default KeyValueIterator, AGG> findSessions(final K key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + throw new UnsupportedOperationException("Moved from SessionStore"); + } + + /** + * Fetch any sessions with the matching key and the sessions end is ≥ earliestSessionEndTime and the sessions + * start is ≤ latestSessionStartTime iterating from latest to earliest. + *

+ * This iterator must be closed after use. + * + * @param key the key to return sessions for + * @param earliestSessionEndTime the end timestamp of the earliest session to search for + * @param latestSessionStartTime the end timestamp of the latest session to search for + * @return backward iterator of sessions with the matching key and aggregated values + * @throws NullPointerException If null is used for key. + */ + default KeyValueIterator, AGG> backwardFindSessions(final K key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + throw new UnsupportedOperationException(); + } + + /** + * Fetch any sessions in the given range of keys and the sessions end is ≥ earliestSessionEndTime and the sessions + * start is ≤ latestSessionStartTime iterating from earliest to latest. + *

+ * This iterator must be closed after use. + * + * @param keyFrom The first key that could be in the range + * @param keyTo The last key that could be in the range + * @param earliestSessionEndTime the end timestamp of the earliest session to search for + * @param latestSessionStartTime the end timestamp of the latest session to search for + * @return iterator of sessions with the matching keys and aggregated values + * @throws NullPointerException If null is used for any key. + */ + default KeyValueIterator, AGG> findSessions(final K keyFrom, + final K keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + throw new UnsupportedOperationException("Moved from SessionStore"); + } + + + /** + * Fetch any sessions in the given range of keys and the sessions end is ≥ earliestSessionEndTime and the sessions + * start is ≤ latestSessionStartTime iterating from latest to earliest. + *

+ * This iterator must be closed after use. + * + * @param keyFrom The first key that could be in the range + * @param keyTo The last key that could be in the range + * @param earliestSessionEndTime the end timestamp of the earliest session to search for + * @param latestSessionStartTime the end timestamp of the latest session to search for + * @return backward iterator of sessions with the matching keys and aggregated values + * @throws NullPointerException If null is used for any key. + */ + default KeyValueIterator, AGG> backwardFindSessions(final K keyFrom, + final K keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + throw new UnsupportedOperationException(); + } + + /** + * Get the value of key from a single session. + * + * @param key the key to fetch + * @param startTime start timestamp of the session + * @param endTime end timestamp of the session + * @return The value or {@code null} if no session associated with the key can be found + * @throws NullPointerException If {@code null} is used for any key. + */ + default AGG fetchSession(final K key, final long startTime, final long endTime) { + throw new UnsupportedOperationException("Moved from SessionStore"); + } + + /** + * Retrieve all aggregated sessions for the provided key. + * This iterator must be closed after use. + *

* For each key, the iterator guarantees ordering of sessions, starting from the oldest/earliest * available session to the newest/latest session. * - * @param key record key to find aggregated session values for - * @return KeyValueIterator containing all sessions for the provided key. - * @throws NullPointerException If null is used for key. - * + * @param key record key to find aggregated session values for + * @return KeyValueIterator containing all sessions for the provided key. + * @throws NullPointerException If null is used for key. */ KeyValueIterator, AGG> fetch(final K key); /** - * Retrieve all aggregated sessions for the given range of keys. + * Retrieve all aggregated sessions for the provided key. * This iterator must be closed after use. + *

+ * For each key, the iterator guarantees ordering of sessions, starting from the newest/latest + * available session to the oldest/earliest session. * + * @param key record key to find aggregated session values for + * @return backward KeyValueIterator containing all sessions for the provided key. + * @throws NullPointerException If null is used for key. + */ + default KeyValueIterator, AGG> backwardFetch(final K key) { + throw new UnsupportedOperationException(); + } + + /** + * Retrieve all aggregated sessions for the given range of keys. + * This iterator must be closed after use. + *

* For each key, the iterator guarantees ordering of sessions, starting from the oldest/earliest * available session to the newest/latest session. * - * @param from first key in the range to find aggregated session values for - * @param to last key in the range to find aggregated session values for - * @return KeyValueIterator containing all sessions for the provided key. - * @throws NullPointerException If null is used for any of the keys. + * @param from first key in the range to find aggregated session values for + * @param to last key in the range to find aggregated session values for + * @return KeyValueIterator containing all sessions for the provided key. + * @throws NullPointerException If null is used for any of the keys. */ KeyValueIterator, AGG> fetch(final K from, final K to); + + /** + * Retrieve all aggregated sessions for the given range of keys. + * This iterator must be closed after use. + *

+ * For each key, the iterator guarantees ordering of sessions, starting from the newest/latest + * available session to the oldest/earliest session. + * + * @param from first key in the range to find aggregated session values for + * @param to last key in the range to find aggregated session values for + * @return backward KeyValueIterator containing all sessions for the provided key. + * @throws NullPointerException If null is used for any of the keys. + */ + default KeyValueIterator, AGG> backwardFetch(final K from, final K to) { + throw new UnsupportedOperationException(); + } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java index faaa751489af4..35e14e6407fe7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java @@ -34,48 +34,9 @@ */ public interface SessionStore extends StateStore, ReadOnlySessionStore { - /** - * Fetch any sessions with the matching key and the sessions end is ≥ earliestSessionEndTime and the sessions - * start is ≤ latestSessionStartTime - * - * This iterator must be closed after use. - * - * @param key the key to return sessions for - * @param earliestSessionEndTime the end timestamp of the earliest session to search for - * @param latestSessionStartTime the end timestamp of the latest session to search for - * @return iterator of sessions with the matching key and aggregated values - * @throws NullPointerException If null is used for key. - */ - KeyValueIterator, AGG> findSessions(final K key, final long earliestSessionEndTime, final long latestSessionStartTime); - - /** - * Fetch any sessions in the given range of keys and the sessions end is ≥ earliestSessionEndTime and the sessions - * start is ≤ latestSessionStartTime - * - * This iterator must be closed after use. - * - * @param keyFrom The first key that could be in the range - * @param keyTo The last key that could be in the range - * @param earliestSessionEndTime the end timestamp of the earliest session to search for - * @param latestSessionStartTime the end timestamp of the latest session to search for - * @return iterator of sessions with the matching keys and aggregated values - * @throws NullPointerException If null is used for any key. - */ - KeyValueIterator, AGG> findSessions(final K keyFrom, final K keyTo, final long earliestSessionEndTime, final long latestSessionStartTime); - - /** - * Get the value of key from a single session. - * - * @param key the key to fetch - * @param startTime start timestamp of the session - * @param endTime end timestamp of the session - * @return The value or {@code null} if no session associated with the key can be found - * @throws NullPointerException If {@code null} is used for any key. - */ - AGG fetchSession(final K key, final long startTime, final long endTime); - /** * Remove the session aggregated with provided {@link Windowed} key from the store + * * @param sessionKey key of the session to remove * @throws NullPointerException If null is used for sessionKey. */ @@ -83,6 +44,7 @@ public interface SessionStore extends StateStore, ReadOnlySessionStore, byte[]> findSessions(final Bytes key, final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime) : context.cache().range(cacheName, - cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), - cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) + cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), + cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) ); - final KeyValueIterator, byte[]> storeIterator = wrapped().findSessions(key, - earliestSessionEndTime, - latestSessionStartTime); + final KeyValueIterator, byte[]> storeIterator = + wrapped().findSessions(key, earliestSessionEndTime, latestSessionStartTime); + final HasNextCondition hasNextCondition = + keySchema.hasNextCondition(key, key, earliestSessionEndTime, latestSessionStartTime); + final PeekingKeyValueIterator filteredCacheIterator = + new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); + return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, false); + } + + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + validateStoreOpen(); + + final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? + new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime) : + context.cache().reverseRange(cacheName, + cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), + cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) + ); + + final KeyValueIterator, byte[]> storeIterator = wrapped().backwardFindSessions(key, + earliestSessionEndTime, + latestSessionStartTime); final HasNextCondition hasNextCondition = keySchema.hasNextCondition(key, - key, - earliestSessionEndTime, - latestSessionStartTime); + key, + earliestSessionEndTime, + latestSessionStartTime); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); - return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction); + return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, true); } @Override @@ -190,13 +212,42 @@ public KeyValueIterator, byte[]> findSessions(final Bytes keyFro final KeyValueIterator, byte[]> storeIterator = wrapped().findSessions( keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime ); + final HasNextCondition hasNextCondition = + keySchema.hasNextCondition(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + final PeekingKeyValueIterator filteredCacheIterator = + new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); + return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, true); + } + + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes keyFrom, + final Bytes keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + if (keyFrom.compareTo(keyTo) > 0) { + LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. " + + "This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " + + "Note that the built-in numerical serdes do not follow this for negative numbers"); + return KeyValueIterators.emptyIterator(); + } + + validateStoreOpen(); + + final Bytes cacheKeyFrom = cacheFunction.cacheKey(keySchema.lowerRange(keyFrom, earliestSessionEndTime)); + final Bytes cacheKeyTo = cacheFunction.cacheKey(keySchema.upperRange(keyTo, latestSessionStartTime)); + final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = context.cache().reverseRange(cacheName, cacheKeyFrom, cacheKeyTo); + + final KeyValueIterator, byte[]> storeIterator = wrapped().backwardFindSessions( + keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime + ); final HasNextCondition hasNextCondition = keySchema.hasNextCondition(keyFrom, - keyTo, - earliestSessionEndTime, - latestSessionStartTime); + keyTo, + earliestSessionEndTime, + latestSessionStartTime); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); - return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction); + return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, false); } @Override @@ -223,6 +274,12 @@ public KeyValueIterator, byte[]> fetch(final Bytes key) { return findSessions(key, 0, Long.MAX_VALUE); } + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes key) { + Objects.requireNonNull(key, "key cannot be null"); + return backwardFindSessions(key, 0, Long.MAX_VALUE); + } + @Override public KeyValueIterator, byte[]> fetch(final Bytes from, final Bytes to) { @@ -231,6 +288,14 @@ public KeyValueIterator, byte[]> fetch(final Bytes from, return findSessions(from, to, 0, Long.MAX_VALUE); } + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes from, + final Bytes to) { + Objects.requireNonNull(from, "from cannot be null"); + Objects.requireNonNull(to, "to cannot be null"); + return backwardFindSessions(from, to, 0, Long.MAX_VALUE); + } + public void flush() { context.cache().flush(cacheName); wrapped().flush(); @@ -249,7 +314,7 @@ public void close() { ); if (!suppressed.isEmpty()) { throwSuppressed("Caught an exception while closing caching session store for store " + name(), - suppressed); + suppressed); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStore.java index cc586d3ba1a90..a41bec43d4e5d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStore.java @@ -49,11 +49,21 @@ public KeyValueIterator, byte[]> findSessions(final Bytes key, f return wrapped().findSessions(key, earliestSessionEndTime, latestSessionStartTime); } + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes key, final long earliestSessionEndTime, final long latestSessionStartTime) { + return wrapped().backwardFindSessions(key, earliestSessionEndTime, latestSessionStartTime); + } + @Override public KeyValueIterator, byte[]> findSessions(final Bytes keyFrom, final Bytes keyTo, final long earliestSessionEndTime, final long latestSessionStartTime) { return wrapped().findSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); } + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes keyFrom, final Bytes keyTo, final long earliestSessionEndTime, final long latestSessionStartTime) { + return wrapped().backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + } + @Override public void remove(final Windowed sessionKey) { wrapped().remove(sessionKey); @@ -71,11 +81,21 @@ public byte[] fetchSession(final Bytes key, final long startTime, final long end return wrapped().fetchSession(key, startTime, endTime); } + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes key) { + return wrapped().backwardFetch(key); + } + @Override public KeyValueIterator, byte[]> fetch(final Bytes key) { return wrapped().fetch(key); } + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes from, final Bytes to) { + return wrapped().backwardFetch(from, to); + } + @Override public KeyValueIterator, byte[]> fetch(final Bytes from, final Bytes to) { return wrapped().fetch(from, to); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java index 63d551c1d089e..46f3e115885de 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java @@ -42,6 +42,123 @@ public CompositeReadOnlySessionStore(final StateStoreProvider storeProvider, this.storeName = storeName; } + @Override + public KeyValueIterator, V> findSessions(final K key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(key, "key can't be null"); + final List> stores = storeProvider.stores(storeName, queryableStoreType); + for (final ReadOnlySessionStore store : stores) { + try { + final KeyValueIterator, V> result = store.findSessions(key, earliestSessionEndTime, latestSessionStartTime); + if (!result.hasNext()) { + result.close(); + } else { + return result; + } + } catch (final InvalidStateStoreException ise) { + throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); + } + } + return KeyValueIterators.emptyIterator(); + } + + @Override + public KeyValueIterator, V> backwardFindSessions(final K key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(key, "key can't be null"); + final List> stores = storeProvider.stores(storeName, queryableStoreType); + for (final ReadOnlySessionStore store : stores) { + try { + final KeyValueIterator, V> result = store.backwardFindSessions(key, earliestSessionEndTime, latestSessionStartTime); + if (!result.hasNext()) { + result.close(); + } else { + return result; + } + } catch (final InvalidStateStoreException ise) { + throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); + } + } + return KeyValueIterators.emptyIterator(); + } + + @Override + public KeyValueIterator, V> findSessions(final K keyFrom, + final K keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(keyFrom, "from can't be null"); + Objects.requireNonNull(keyTo, "to can't be null"); + final List> stores = storeProvider.stores(storeName, queryableStoreType); + for (final ReadOnlySessionStore store : stores) { + try { + final KeyValueIterator, V> result = store.findSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + if (!result.hasNext()) { + result.close(); + } else { + return result; + } + } catch (final InvalidStateStoreException ise) { + throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); + } + } + return KeyValueIterators.emptyIterator(); + } + + @Override + public KeyValueIterator, V> backwardFindSessions(final K keyFrom, + final K keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(keyFrom, "from can't be null"); + Objects.requireNonNull(keyTo, "to can't be null"); + final List> stores = storeProvider.stores(storeName, queryableStoreType); + for (final ReadOnlySessionStore store : stores) { + try { + final KeyValueIterator, V> result = store.backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + if (!result.hasNext()) { + result.close(); + } else { + return result; + } + } catch (final InvalidStateStoreException ise) { + throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); + } + } + return KeyValueIterators.emptyIterator(); + } + + @Override + public V fetchSession(final K key, final long startTime, final long endTime) { + Objects.requireNonNull(key, "key can't be null"); + final List> stores = storeProvider.stores(storeName, queryableStoreType); + for (final ReadOnlySessionStore store : stores) { + try { + return store.fetchSession(key, startTime, endTime); + } catch (final InvalidStateStoreException ise) { + throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); + } + } + return null; + } + @Override public KeyValueIterator, V> fetch(final K key) { Objects.requireNonNull(key, "key can't be null"); @@ -56,9 +173,31 @@ public KeyValueIterator, V> fetch(final K key) { } } catch (final InvalidStateStoreException ise) { throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); + } + } + return KeyValueIterators.emptyIterator(); + } + + @Override + public KeyValueIterator, V> backwardFetch(final K key) { + Objects.requireNonNull(key, "key can't be null"); + final List> stores = storeProvider.stores(storeName, queryableStoreType); + for (final ReadOnlySessionStore store : stores) { + try { + final KeyValueIterator, V> result = store.backwardFetch(key); + if (!result.hasNext()) { + result.close(); + } else { + return result; + } + } catch (final InvalidStateStoreException ise) { + throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); } } return KeyValueIterators.emptyIterator(); @@ -70,8 +209,19 @@ public KeyValueIterator, V> fetch(final K from, final K to) { Objects.requireNonNull(to, "to can't be null"); final NextIteratorFunction, V, ReadOnlySessionStore> nextIteratorFunction = store -> store.fetch(from, to); return new DelegatingPeekingKeyValueIterator<>(storeName, - new CompositeKeyValueIterator<>( - storeProvider.stores(storeName, queryableStoreType).iterator(), - nextIteratorFunction)); + new CompositeKeyValueIterator<>( + storeProvider.stores(storeName, queryableStoreType).iterator(), + nextIteratorFunction)); + } + + @Override + public KeyValueIterator, V> backwardFetch(final K from, final K to) { + Objects.requireNonNull(from, "from can't be null"); + Objects.requireNonNull(to, "to can't be null"); + final NextIteratorFunction, V, ReadOnlySessionStore> nextIteratorFunction = store -> store.backwardFetch(from, to); + return new DelegatingPeekingKeyValueIterator<>(storeName, + new CompositeKeyValueIterator<>( + storeProvider.stores(storeName, queryableStoreType).iterator(), + nextIteratorFunction)); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java index e4fda06682c86..f7bea499d20a7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java @@ -55,7 +55,7 @@ public class InMemorySessionStore implements SessionStore { private final long retentionPeriod; private final ConcurrentNavigableMap>> endTimeMap = new ConcurrentSkipListMap<>(); - private final Set openIterators = ConcurrentHashMap.newKeySet(); + private final Set openIterators = ConcurrentHashMap.newKeySet(); private volatile boolean open = false; @@ -164,10 +164,28 @@ public KeyValueIterator, byte[]> findSessions(final Bytes key, removeExpiredSegments(); - return registerNewIterator(key, - key, - latestSessionStartTime, - endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator()); + return registerNewIterator( + key, + key, + latestSessionStartTime, + endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator(), + true); + } + + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(key, "key cannot be null"); + + removeExpiredSegments(); + + return registerNewIterator( + key, + key, + latestSessionStartTime, + endTimeMap.tailMap(earliestSessionEndTime, true).descendingMap().entrySet().iterator(), + false); } @Override @@ -188,10 +206,38 @@ public KeyValueIterator, byte[]> findSessions(final Bytes keyFro return KeyValueIterators.emptyIterator(); } - return registerNewIterator(keyFrom, - keyTo, - latestSessionStartTime, - endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator()); + return registerNewIterator( + keyFrom, + keyTo, + latestSessionStartTime, + endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator(), + true); + } + + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes keyFrom, + final Bytes keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(keyFrom, "from key cannot be null"); + Objects.requireNonNull(keyTo, "to key cannot be null"); + + removeExpiredSegments(); + + if (keyFrom.compareTo(keyTo) > 0) { + LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. " + + "This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " + + "Note that the built-in numerical serdes do not follow this for negative numbers"); + return KeyValueIterators.emptyIterator(); + } + + return registerNewIterator( + keyFrom, + keyTo, + latestSessionStartTime, + endTimeMap.tailMap(earliestSessionEndTime, true).descendingMap().entrySet().iterator(), + false); } @Override @@ -201,7 +247,26 @@ public KeyValueIterator, byte[]> fetch(final Bytes key) { removeExpiredSegments(); - return registerNewIterator(key, key, Long.MAX_VALUE, endTimeMap.entrySet().iterator()); + return registerNewIterator( + key, + key, + Long.MAX_VALUE, endTimeMap.entrySet().iterator(), + true); + } + + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes key) { + + Objects.requireNonNull(key, "key cannot be null"); + + removeExpiredSegments(); + + return registerNewIterator( + key, + key, + Long.MAX_VALUE, + endTimeMap.descendingMap().entrySet().iterator(), + false); } @Override @@ -212,8 +277,17 @@ public KeyValueIterator, byte[]> fetch(final Bytes from, final B removeExpiredSegments(); + return registerNewIterator(from, to, Long.MAX_VALUE, endTimeMap.entrySet().iterator(), false); + } - return registerNewIterator(from, to, Long.MAX_VALUE, endTimeMap.entrySet().iterator()); + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes from, final Bytes to) { + Objects.requireNonNull(from, "from key cannot be null"); + Objects.requireNonNull(to, "to key cannot be null"); + + removeExpiredSegments(); + + return registerNewIterator(from, to, Long.MAX_VALUE, endTimeMap.descendingMap().entrySet().iterator(), true); } @Override @@ -258,8 +332,16 @@ private void removeExpiredSegments() { private InMemorySessionStoreIterator registerNewIterator(final Bytes keyFrom, final Bytes keyTo, final long latestSessionStartTime, - final Iterator>>> endTimeIterator) { - final InMemorySessionStoreIterator iterator = new InMemorySessionStoreIterator(keyFrom, keyTo, latestSessionStartTime, endTimeIterator, openIterators::remove); + final Iterator>>> endTimeIterator, + final boolean forward) { + final InMemorySessionStoreIterator iterator = + new InMemorySessionStoreIterator( + keyFrom, + keyTo, + latestSessionStartTime, + endTimeIterator, + openIterators::remove, + forward); openIterators.add(iterator); return iterator; } @@ -270,7 +352,6 @@ interface ClosingCallback { private static class InMemorySessionStoreIterator implements KeyValueIterator, byte[]> { - private final Iterator>>> endTimeIterator; private Iterator>> keyIterator; private Iterator> recordIterator; @@ -281,20 +362,24 @@ private static class InMemorySessionStoreIterator implements KeyValueIterator>>> endTimeIterator; private final ClosingCallback callback; + private final boolean forward; + InMemorySessionStoreIterator(final Bytes keyFrom, final Bytes keyTo, final long latestSessionStartTime, final Iterator>>> endTimeIterator, - final ClosingCallback callback) { + final ClosingCallback callback, + final boolean forward) { this.keyFrom = keyFrom; this.keyTo = keyTo; this.latestSessionStartTime = latestSessionStartTime; - this.endTimeIterator = endTimeIterator; this.callback = callback; + this.forward = forward; setAllIterators(); } @@ -365,7 +450,18 @@ private void setAllIterators() { while (endTimeIterator.hasNext()) { final Entry>> nextEndTimeEntry = endTimeIterator.next(); currentEndTime = nextEndTimeEntry.getKey(); - keyIterator = nextEndTimeEntry.getValue().subMap(keyFrom, true, keyTo, true).entrySet().iterator(); + final Set>> entries; + if (forward) { + entries = nextEndTimeEntry.getValue() + .subMap(keyFrom, true, keyTo, true) + .entrySet(); + } else { + entries = nextEndTimeEntry.getValue() + .subMap(keyFrom, true, keyTo, true) + .descendingMap() + .entrySet(); + } + keyIterator = entries.iterator(); if (setInnerIterators()) { return; @@ -382,9 +478,20 @@ private boolean setInnerIterators() { currentKey = nextKeyEntry.getKey(); if (latestSessionStartTime == Long.MAX_VALUE) { - recordIterator = nextKeyEntry.getValue().entrySet().iterator(); + final Set> entries; + if (forward) entries = nextKeyEntry.getValue().descendingMap().entrySet(); + else entries = nextKeyEntry.getValue().entrySet(); + recordIterator = entries.iterator(); } else { - recordIterator = nextKeyEntry.getValue().headMap(latestSessionStartTime, true).entrySet().iterator(); + final Set> entries; + if (forward) entries = nextKeyEntry.getValue() + .headMap(latestSessionStartTime, true) + .descendingMap() + .entrySet(); + else entries = nextKeyEntry.getValue() + .headMap(latestSessionStartTime, true) + .entrySet(); + recordIterator = entries.iterator(); } if (recordIterator.hasNext()) { @@ -400,9 +507,7 @@ private void getNextIterators() { if (setInnerIterators()) { return; } - setAllIterators(); } } - } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java index ff45a418889a9..5a9204071d697 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java @@ -24,16 +24,17 @@ /** * Merges two iterators. Assumes each of them is sorted by key - * */ -class MergedSortedCacheSessionStoreIterator extends AbstractMergedSortedCacheStoreIterator, Windowed, byte[], byte[]> { +class MergedSortedCacheSessionStoreIterator + extends AbstractMergedSortedCacheStoreIterator, Windowed, byte[], byte[]> { private final SegmentedCacheFunction cacheFunction; MergedSortedCacheSessionStoreIterator(final PeekingKeyValueIterator cacheIterator, final KeyValueIterator, byte[]> storeIterator, - final SegmentedCacheFunction cacheFunction) { - super(cacheIterator, storeIterator, true); + final SegmentedCacheFunction cacheFunction, + final boolean forward) { + super(cacheIterator, storeIterator, forward); this.cacheFunction = cacheFunction; } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java index d8ce02a6d18bb..12150838c53d0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java @@ -185,6 +185,17 @@ public KeyValueIterator, V> fetch(final K key) { time); } + @Override + public KeyValueIterator, V> backwardFetch(final K key) { + Objects.requireNonNull(key, "key cannot be null"); + return new MeteredWindowedKeyValueIterator<>( + wrapped().backwardFetch(keyBytes(key)), + fetchSensor, + streamsMetrics, + serdes, + time); + } + @Override public KeyValueIterator, V> fetch(final K from, final K to) { @@ -198,6 +209,19 @@ public KeyValueIterator, V> fetch(final K from, time); } + @Override + public KeyValueIterator, V> backwardFetch(final K from, + final K to) { + Objects.requireNonNull(from, "from cannot be null"); + Objects.requireNonNull(to, "to cannot be null"); + return new MeteredWindowedKeyValueIterator<>( + wrapped().backwardFetch(keyBytes(from), keyBytes(to)), + fetchSensor, + streamsMetrics, + serdes, + time); + } + @Override public KeyValueIterator, V> findSessions(final K key, final long earliestSessionEndTime, @@ -215,6 +239,23 @@ public KeyValueIterator, V> findSessions(final K key, time); } + @Override + public KeyValueIterator, V> backwardFindSessions(final K key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(key, "key cannot be null"); + final Bytes bytesKey = keyBytes(key); + return new MeteredWindowedKeyValueIterator<>( + wrapped().backwardFindSessions( + bytesKey, + earliestSessionEndTime, + latestSessionStartTime), + fetchSensor, + streamsMetrics, + serdes, + time); + } + @Override public KeyValueIterator, V> findSessions(final K keyFrom, final K keyTo, @@ -236,6 +277,27 @@ public KeyValueIterator, V> findSessions(final K keyFrom, time); } + @Override + public KeyValueIterator, V> backwardFindSessions(final K keyFrom, + final K keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + Objects.requireNonNull(keyFrom, "keyFrom cannot be null"); + Objects.requireNonNull(keyTo, "keyTo cannot be null"); + final Bytes bytesKeyFrom = keyBytes(keyFrom); + final Bytes bytesKeyTo = keyBytes(keyTo); + return new MeteredWindowedKeyValueIterator<>( + wrapped().backwardFindSessions( + bytesKeyFrom, + bytesKeyTo, + earliestSessionEndTime, + latestSessionStartTime), + fetchSensor, + streamsMetrics, + serdes, + time); + } + @Override public void flush() { maybeMeasureLatency(super::flush, time, flushSensor); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java index ecbcb5a841f8e..b9fdebeca7c20 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreFacade.java @@ -63,13 +63,6 @@ public WindowStoreIterator backwardFetch(final K key, return new WindowStoreIteratorFacade<>(inner.backwardFetch(key, timeFrom, timeTo)); } - @Override - public WindowStoreIterator backwardFetch(final K key, - final Instant from, - final Instant to) throws IllegalArgumentException { - return new WindowStoreIteratorFacade<>(inner.backwardFetch(key, from, to)); - } - @Override @SuppressWarnings("deprecation") public KeyValueIterator, V> fetch(final K keyFrom, @@ -95,14 +88,6 @@ public KeyValueIterator, V> backwardFetch(final K keyFrom, return new KeyValueIteratorFacade<>(inner.backwardFetch(keyFrom, keyTo, timeFrom, timeTo)); } - @Override - public KeyValueIterator, V> backwardFetch(final K from, - final K to, - final Instant fromTime, - final Instant toTime) throws IllegalArgumentException { - return new KeyValueIteratorFacade<>(inner.backwardFetch(from, to, fromTime, toTime)); - } - @Override @SuppressWarnings("deprecation") public KeyValueIterator, V> fetchAll(final long timeFrom, @@ -122,13 +107,6 @@ public KeyValueIterator, V> backwardFetchAll(final Instant timeFrom, return new KeyValueIteratorFacade<>(inner.backwardFetchAll(timeFrom, timeTo)); } - @Override - public KeyValueIterator, V> backwardFetchAll(final Instant from, - final Instant to) throws IllegalArgumentException { - final KeyValueIterator, ValueAndTimestamp> innerIterator = inner.backwardFetchAll(from, to); - return new KeyValueIteratorFacade<>(innerIterator); - } - @Override public KeyValueIterator, V> all() { return new KeyValueIteratorFacade<>(inner.all()); @@ -139,12 +117,6 @@ public KeyValueIterator, V> backwardAll() { return new KeyValueIteratorFacade<>(inner.backwardAll()); } - @Override - public KeyValueIterator, V> backwardAll() { - final KeyValueIterator, ValueAndTimestamp> innerIterator = inner.backwardAll(); - return new KeyValueIteratorFacade<>(innerIterator); - } - private static class WindowStoreIteratorFacade implements WindowStoreIterator { final KeyValueIterator> innerIterator; diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java index 2f7a211d5266a..e208ac12d56d3 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java @@ -21,7 +21,6 @@ import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.SessionStore; - public class RocksDBSessionStore extends WrappedStateStore implements SessionStore { @@ -42,6 +41,18 @@ public KeyValueIterator, byte[]> findSessions(final Bytes key, return new WrappedSessionStoreIterator(bytesIterator); } + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + final KeyValueIterator bytesIterator = wrapped().backwardFetch( + key, + earliestSessionEndTime, + latestSessionStartTime + ); + return new WrappedSessionStoreIterator(bytesIterator); + } + @Override public KeyValueIterator, byte[]> findSessions(final Bytes keyFrom, final Bytes keyTo, @@ -56,6 +67,20 @@ public KeyValueIterator, byte[]> findSessions(final Bytes keyFro return new WrappedSessionStoreIterator(bytesIterator); } + @Override + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes keyFrom, + final Bytes keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { + final KeyValueIterator bytesIterator = wrapped().backwardFetch( + keyFrom, + keyTo, + earliestSessionEndTime, + latestSessionStartTime + ); + return new WrappedSessionStoreIterator(bytesIterator); + } + @Override public byte[] fetchSession(final Bytes key, final long startTime, final long endTime) { return wrapped().get(SessionKeySchema.toBinary(key, startTime, endTime)); @@ -66,11 +91,21 @@ public KeyValueIterator, byte[]> fetch(final Bytes key) { return findSessions(key, 0, Long.MAX_VALUE); } + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes key) { + return backwardFindSessions(key, 0, Long.MAX_VALUE); + } + @Override public KeyValueIterator, byte[]> fetch(final Bytes from, final Bytes to) { return findSessions(from, to, 0, Long.MAX_VALUE); } + @Override + public KeyValueIterator, byte[]> backwardFetch(final Bytes from, final Bytes to) { + return backwardFindSessions(from, to, 0, Long.MAX_VALUE); + } + @Override public void remove(final Windowed key) { wrapped().remove(SessionKeySchema.toBinary(key)); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java index 326b86945f8e4..f3c452495839b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java @@ -26,7 +26,6 @@ import java.nio.ByteBuffer; import java.util.List; - public class SessionKeySchema implements SegmentedBytesStore.KeySchema { private static final int TIMESTAMP_SIZE = 8; diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java index bb425a9f8e8d6..795515e7de261 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java @@ -66,7 +66,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; - public abstract class AbstractSessionBytesStoreTest { static final long SEGMENT_INTERVAL = 60_000L; @@ -120,16 +119,43 @@ public void shouldPutAndFindSessionsInRange() { final List, Long>> expected = Arrays.asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L)); - try (final KeyValueIterator, Long> values = sessionStore.findSessions(key, 0, 1000L) - ) { + try (final KeyValueIterator, Long> values = + sessionStore.findSessions(key, 0, 1000L)) { assertEquals(new HashSet<>(expected), toSet(values)); } final List, Long>> expected2 = Collections.singletonList(KeyValue.pair(a2, 2L)); - try (final KeyValueIterator, Long> values2 = sessionStore.findSessions(key, 400L, 600L) - ) { + try (final KeyValueIterator, Long> values2 = + sessionStore.findSessions(key, 400L, 600L)) { + assertEquals(new HashSet<>(expected2), toSet(values2)); + } + } + + @Test + public void shouldPutAndBackwardFindSessionsInRange() { + final String key = "a"; + final Windowed a1 = new Windowed<>(key, new SessionWindow(10, 10L)); + final Windowed a2 = new Windowed<>(key, new SessionWindow(500L, 1000L)); + sessionStore.put(a1, 1L); + sessionStore.put(a2, 2L); + sessionStore.put(new Windowed<>(key, new SessionWindow(1500L, 2000L)), 1L); + sessionStore.put(new Windowed<>(key, new SessionWindow(2500L, 3000L)), 2L); + + final List, Long>> expected = + Arrays.asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L)); + + try (final KeyValueIterator, Long> values = + sessionStore.backwardFindSessions(key, 0, 1000L)) { + assertEquals(new HashSet<>(expected), toSet(values)); + } + + final List, Long>> expected2 = + Collections.singletonList(KeyValue.pair(a2, 2L)); + + try (final KeyValueIterator, Long> values2 = + sessionStore.backwardFindSessions(key, 400L, 600L)) { assertEquals(new HashSet<>(expected2), toSet(values2)); } } @@ -154,6 +180,26 @@ public void shouldFetchAllSessionsWithSameRecordKey() { } } + @Test + public void shouldBackwardFetchAllSessionsWithSameRecordKey() { + final List, Long>> expected = Arrays.asList( + KeyValue.pair(new Windowed<>("a", new SessionWindow(0, 0)), 1L), + KeyValue.pair(new Windowed<>("a", new SessionWindow(10, 10)), 2L), + KeyValue.pair(new Windowed<>("a", new SessionWindow(100, 100)), 3L), + KeyValue.pair(new Windowed<>("a", new SessionWindow(1000, 1000)), 4L)); + + for (final KeyValue, Long> kv : expected) { + sessionStore.put(kv.key, kv.value); + } + + // add one that shouldn't appear in the results + sessionStore.put(new Windowed<>("aa", new SessionWindow(0, 0)), 5L); + + try (final KeyValueIterator, Long> values = sessionStore.backwardFetch("a")) { + assertEquals(new HashSet<>(expected), toSet(values)); + } + } + @Test public void shouldFetchAllSessionsWithinKeyRange() { final List, Long>> expected = Arrays.asList( @@ -176,6 +222,28 @@ public void shouldFetchAllSessionsWithinKeyRange() { } } + @Test + public void shouldBackwardFetchAllSessionsWithinKeyRange() { + final List, Long>> expected = Arrays.asList( + KeyValue.pair(new Windowed<>("aa", new SessionWindow(10, 10)), 2L), + KeyValue.pair(new Windowed<>("b", new SessionWindow(1000, 1000)), 4L), + + KeyValue.pair(new Windowed<>("aaa", new SessionWindow(100, 100)), 3L), + KeyValue.pair(new Windowed<>("bb", new SessionWindow(1500, 2000)), 5L)); + + for (final KeyValue, Long> kv : expected) { + sessionStore.put(kv.key, kv.value); + } + + // add some that shouldn't appear in the results + sessionStore.put(new Windowed<>("a", new SessionWindow(0, 0)), 1L); + sessionStore.put(new Windowed<>("bbb", new SessionWindow(2500, 3000)), 6L); + + try (final KeyValueIterator, Long> values = sessionStore.backwardFetch("aa", "bb")) { + assertEquals(new HashSet<>(expected), toSet(values)); + } + } + @Test public void shouldFetchExactSession() { sessionStore.put(new Windowed<>("a", new SessionWindow(0, 4)), 1L); @@ -208,6 +276,21 @@ public void shouldFindValuesWithinMergingSessionWindowRange() { } } + @Test + public void shouldBackwardFindValuesWithinMergingSessionWindowRange() { + final String key = "a"; + sessionStore.put(new Windowed<>(key, new SessionWindow(0L, 0L)), 1L); + sessionStore.put(new Windowed<>(key, new SessionWindow(1000L, 1000L)), 2L); + + final List, Long>> expected = Arrays.asList( + KeyValue.pair(new Windowed<>(key, new SessionWindow(0L, 0L)), 1L), + KeyValue.pair(new Windowed<>(key, new SessionWindow(1000L, 1000L)), 2L)); + + try (final KeyValueIterator, Long> results = sessionStore.backwardFindSessions(key, -1, 1000L)) { + assertEquals(new HashSet<>(expected), toSet(results)); + } + } + @Test public void shouldRemove() { sessionStore.put(new Windowed<>("a", new SessionWindow(0, 1000)), 1L); @@ -261,6 +344,27 @@ public void shouldFindSessionsToMerge() { } } + @Test + public void shouldBackwardFindSessionsToMerge() { + final Windowed session1 = new Windowed<>("a", new SessionWindow(0, 100)); + final Windowed session2 = new Windowed<>("a", new SessionWindow(101, 200)); + final Windowed session3 = new Windowed<>("a", new SessionWindow(201, 300)); + final Windowed session4 = new Windowed<>("a", new SessionWindow(301, 400)); + final Windowed session5 = new Windowed<>("a", new SessionWindow(401, 500)); + sessionStore.put(session1, 1L); + sessionStore.put(session2, 2L); + sessionStore.put(session3, 3L); + sessionStore.put(session4, 4L); + sessionStore.put(session5, 5L); + + final List, Long>> expected = + Arrays.asList(KeyValue.pair(session2, 2L), KeyValue.pair(session3, 3L)); + + try (final KeyValueIterator, Long> results = sessionStore.backwardFindSessions("a", 150, 300)) { + assertEquals(new HashSet<>(expected), toSet(results)); + } + } + @Test public void shouldFetchExactKeys() { sessionStore = buildSessionStore(0x7a00000000000000L, Serdes.String(), Serdes.Long()); @@ -298,6 +402,43 @@ public void shouldFetchExactKeys() { } } + @Test + public void shouldBackwardFetchExactKeys() { + sessionStore = buildSessionStore(0x7a00000000000000L, Serdes.String(), Serdes.Long()); + sessionStore.init(context, sessionStore); + + sessionStore.put(new Windowed<>("a", new SessionWindow(0, 0)), 1L); + sessionStore.put(new Windowed<>("aa", new SessionWindow(0, 10)), 2L); + sessionStore.put(new Windowed<>("a", new SessionWindow(10, 20)), 3L); + sessionStore.put(new Windowed<>("aa", new SessionWindow(10, 20)), 4L); + sessionStore.put(new Windowed<>("a", + new SessionWindow(0x7a00000000000000L - 2, 0x7a00000000000000L - 1)), 5L); + + try (final KeyValueIterator, Long> iterator = + sessionStore.backwardFindSessions("a", 0, Long.MAX_VALUE) + ) { + assertThat(valuesToSet(iterator), equalTo(new HashSet<>(asList(1L, 3L, 5L)))); + } + + try (final KeyValueIterator, Long> iterator = + sessionStore.backwardFindSessions("aa", 0, Long.MAX_VALUE) + ) { + assertThat(valuesToSet(iterator), equalTo(new HashSet<>(asList(2L, 4L)))); + } + + try (final KeyValueIterator, Long> iterator = + sessionStore.backwardFindSessions("a", "aa", 0, Long.MAX_VALUE) + ) { + assertThat(valuesToSet(iterator), equalTo(new HashSet<>(asList(1L, 2L, 3L, 4L, 5L)))); + } + + try (final KeyValueIterator, Long> iterator = + sessionStore.backwardFindSessions("a", "aa", 10, 0) + ) { + assertThat(valuesToSet(iterator), equalTo(new HashSet<>(Collections.singletonList(2L)))); + } + } + @Test public void shouldFetchAndIterateOverExactBinaryKeys() { final SessionStore sessionStore = @@ -327,6 +468,35 @@ public void shouldFetchAndIterateOverExactBinaryKeys() { assertThat(valuesToSet(sessionStore.findSessions(key3, 0L, Long.MAX_VALUE)), equalTo(expectedKey3)); } + @Test + public void shouldBackwardFetchAndIterateOverExactBinaryKeys() { + final SessionStore sessionStore = + buildSessionStore(RETENTION_PERIOD, Serdes.Bytes(), Serdes.String()); + + sessionStore.init(context, sessionStore); + + final Bytes key1 = Bytes.wrap(new byte[] {0}); + final Bytes key2 = Bytes.wrap(new byte[] {0, 0}); + final Bytes key3 = Bytes.wrap(new byte[] {0, 0, 0}); + + sessionStore.put(new Windowed<>(key1, new SessionWindow(1, 100)), "1"); + sessionStore.put(new Windowed<>(key2, new SessionWindow(2, 100)), "2"); + sessionStore.put(new Windowed<>(key3, new SessionWindow(3, 100)), "3"); + sessionStore.put(new Windowed<>(key1, new SessionWindow(4, 100)), "4"); + sessionStore.put(new Windowed<>(key2, new SessionWindow(5, 100)), "5"); + sessionStore.put(new Windowed<>(key3, new SessionWindow(6, 100)), "6"); + sessionStore.put(new Windowed<>(key1, new SessionWindow(7, 100)), "7"); + sessionStore.put(new Windowed<>(key2, new SessionWindow(8, 100)), "8"); + sessionStore.put(new Windowed<>(key3, new SessionWindow(9, 100)), "9"); + + final Set expectedKey1 = new HashSet<>(asList("1", "4", "7")); + assertThat(valuesToSet(sessionStore.backwardFindSessions(key1, 0L, Long.MAX_VALUE)), equalTo(expectedKey1)); + final Set expectedKey2 = new HashSet<>(asList("2", "5", "8")); + assertThat(valuesToSet(sessionStore.backwardFindSessions(key2, 0L, Long.MAX_VALUE)), equalTo(expectedKey2)); + final Set expectedKey3 = new HashSet<>(asList("3", "6", "9")); + assertThat(valuesToSet(sessionStore.backwardFindSessions(key3, 0L, Long.MAX_VALUE)), equalTo(expectedKey3)); + } + @Test public void testIteratorPeek() { sessionStore.put(new Windowed<>("a", new SessionWindow(0, 0)), 1L); @@ -342,6 +512,21 @@ public void testIteratorPeek() { assertFalse(iterator.hasNext()); } + @Test + public void testIteratorPeekBackward() { + sessionStore.put(new Windowed<>("a", new SessionWindow(0, 0)), 1L); + sessionStore.put(new Windowed<>("aa", new SessionWindow(0, 10)), 2L); + sessionStore.put(new Windowed<>("a", new SessionWindow(10, 20)), 3L); + sessionStore.put(new Windowed<>("aa", new SessionWindow(10, 20)), 4L); + + final KeyValueIterator, Long> iterator = sessionStore.backwardFindSessions("a", 0L, 20); + + assertEquals(iterator.peekNextKey(), new Windowed<>("a", new SessionWindow(10L, 20L))); + assertEquals(iterator.peekNextKey(), iterator.next().key); + assertEquals(iterator.peekNextKey(), iterator.next().key); + assertFalse(iterator.hasNext()); + } + @Test public void shouldRestore() { final List, Long>> expected = Arrays.asList( diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java index f36629d1aca57..7debda23c9b77 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java @@ -81,7 +81,6 @@ public class CachingSessionStoreTest { private SessionStore underlyingStore = new InMemorySessionStore("store-name", Long.MAX_VALUE, "metric-scope"); - private InternalMockProcessorContext context; private CachingSessionStore cachingStore; private ThreadCache cache; @@ -131,6 +130,21 @@ public void shouldPutFetchAllKeysFromCache() { assertFalse(all.hasNext()); } + @Test + public void shouldPutBackwardFetchAllKeysFromCache() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(0, 0)), "1".getBytes()); + + assertEquals(3, cache.size()); + + final KeyValueIterator, byte[]> all = cachingStore.backwardFindSessions(keyA, keyB, 0, 0); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyA, new SessionWindow(0, 0)), "1"); + assertFalse(all.hasNext()); + } + @Test public void shouldCloseWrappedStoreAndCacheAfterErrorDuringCacheFlush() { setUpCloseTests(); @@ -185,7 +199,7 @@ private void setUpCloseTests() { EasyMock.replay(underlyingStore); cachingStore = new CachingSessionStore(underlyingStore, SEGMENT_INTERVAL); cache = EasyMock.niceMock(ThreadCache.class); - context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); + InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); context.setRecordContext(new ProcessorRecordContext(10, 0, 0, TOPIC, null)); cachingStore.init((StateStoreContext) context, cachingStore); } @@ -204,6 +218,20 @@ public void shouldPutFetchRangeFromCache() { assertFalse(some.hasNext()); } + @Test + public void shouldPutBackwardFetchRangeFromCache() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(0, 0)), "1".getBytes()); + + assertEquals(3, cache.size()); + + final KeyValueIterator, byte[]> some = cachingStore.backwardFindSessions(keyAA, keyB, 0, 0); + verifyWindowedKeyValue(some.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(some.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); + assertFalse(some.hasNext()); + } + @Test public void shouldFetchAllSessionsWithSameRecordKey() { final List, byte[]>> expected = asList( @@ -223,6 +251,26 @@ public void shouldFetchAllSessionsWithSameRecordKey() { verifyKeyValueList(expected, results); } + @Test + public void shouldBackwardFetchAllSessionsWithSameRecordKey() { + final List, byte[]>> expected = asList( + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(10, 10)), "2".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(100, 100)), "3".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(1000, 1000)), "4".getBytes()) + ); + for (final KeyValue, byte[]> kv : expected) { + cachingStore.put(kv.key, kv.value); + } + + // add one that shouldn't appear in the results + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "5".getBytes()); + + final List, byte[]>> results = toList(cachingStore.backwardFetch(keyA)); + Collections.reverse(results); + verifyKeyValueList(expected, results); + } + @Test public void shouldFlushItemsToStoreOnEviction() { final List, byte[]>> added = addSessionsUntilOverflow("a", "b", "c", "d"); @@ -301,6 +349,29 @@ public void shouldFetchRangeCorrectlyAcrossSegments() { assertEquals(mkSet(a1, a2, a3, aa1, aa3), keys); } + @Test + public void shouldBackwardFetchRangeCorrectlyAcrossSegments() { + final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed aa1 = new Windowed<>(keyAA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); + final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + final Windowed aa3 = new Windowed<>(keyAA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + cachingStore.put(a1, "1".getBytes()); + cachingStore.put(aa1, "1".getBytes()); + cachingStore.put(a2, "2".getBytes()); + cachingStore.put(a3, "3".getBytes()); + cachingStore.put(aa3, "3".getBytes()); + + final KeyValueIterator, byte[]> rangeResults = + cachingStore.backwardFindSessions(keyA, keyAA, 0, SEGMENT_INTERVAL * 2); + final Set> keys = new HashSet<>(); + while (rangeResults.hasNext()) { + keys.add(rangeResults.next().key); + } + rangeResults.close(); + assertEquals(mkSet(a1, a2, a3, aa1, aa3), keys); + } + @Test public void shouldSetFlushListener() { assertTrue(cachingStore.setFlushListener(null, true)); @@ -456,68 +527,68 @@ public void shouldClearNamespaceCacheOnClose() { assertEquals(0, cache.size()); } - @Test(expected = InvalidStateStoreException.class) + @Test public void shouldThrowIfTryingToFetchFromClosedCachingStore() { cachingStore.close(); - cachingStore.fetch(keyA); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.fetch(keyA)); } - @Test(expected = InvalidStateStoreException.class) + @Test public void shouldThrowIfTryingToFindMergeSessionFromClosedCachingStore() { cachingStore.close(); - cachingStore.findSessions(keyA, 0, Long.MAX_VALUE); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.findSessions(keyA, 0, Long.MAX_VALUE)); } - @Test(expected = InvalidStateStoreException.class) + @Test public void shouldThrowIfTryingToRemoveFromClosedCachingStore() { cachingStore.close(); - cachingStore.remove(new Windowed<>(keyA, new SessionWindow(0, 0))); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.remove(new Windowed<>(keyA, new SessionWindow(0, 0)))); } - @Test(expected = InvalidStateStoreException.class) + @Test public void shouldThrowIfTryingToPutIntoClosedCachingStore() { cachingStore.close(); - cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes())); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnFindSessionsNullKey() { - cachingStore.findSessions(null, 1L, 2L); + assertThrows(NullPointerException.class, () -> cachingStore.findSessions(null, 1L, 2L)); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnFindSessionsNullFromKey() { - cachingStore.findSessions(null, keyA, 1L, 2L); + assertThrows(NullPointerException.class, () -> cachingStore.findSessions(null, keyA, 1L, 2L)); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnFindSessionsNullToKey() { - cachingStore.findSessions(keyA, null, 1L, 2L); + assertThrows(NullPointerException.class, () -> cachingStore.findSessions(keyA, null, 1L, 2L)); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnFetchNullFromKey() { - cachingStore.fetch(null, keyA); + assertThrows(NullPointerException.class, () -> cachingStore.fetch(null, keyA)); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnFetchNullToKey() { - cachingStore.fetch(keyA, null); + assertThrows(NullPointerException.class, () -> cachingStore.fetch(keyA, null)); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnFetchNullKey() { - cachingStore.fetch(null); + assertThrows(NullPointerException.class, () -> cachingStore.fetch(null)); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnRemoveNullKey() { - cachingStore.remove(null); + assertThrows(NullPointerException.class, () -> cachingStore.remove(null)); } - @Test(expected = NullPointerException.class) + @Test public void shouldThrowNullPointerExceptionOnPutNullKey() { - cachingStore.put(null, "1".getBytes()); + assertThrows(NullPointerException.class, () -> cachingStore.put(null, "1".getBytes())); } @Test diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java index 426a33484fb56..1ff3153813a8f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java @@ -111,6 +111,16 @@ public void shouldDelegateToUnderlyingStoreWhenFetching() { EasyMock.verify(inner); } + @Test + public void shouldDelegateToUnderlyingStoreWhenBackwardFetching() { + EasyMock.expect(inner.backwardFetch(bytesKey)).andReturn(KeyValueIterators.emptyIterator()); + + init(); + + store.backwardFetch(bytesKey); + EasyMock.verify(inner); + } + @Test public void shouldDelegateToUnderlyingStoreWhenFetchingRange() { EasyMock.expect(inner.fetch(bytesKey, bytesKey)).andReturn(KeyValueIterators.emptyIterator()); @@ -121,6 +131,16 @@ public void shouldDelegateToUnderlyingStoreWhenFetchingRange() { EasyMock.verify(inner); } + @Test + public void shouldDelegateToUnderlyingStoreWhenBackwardFetchingRange() { + EasyMock.expect(inner.backwardFetch(bytesKey, bytesKey)).andReturn(KeyValueIterators.emptyIterator()); + + init(); + + store.backwardFetch(bytesKey, bytesKey); + EasyMock.verify(inner); + } + @Test public void shouldDelegateToUnderlyingStoreWhenFindingSessions() { EasyMock.expect(inner.findSessions(bytesKey, 0, 1)).andReturn(KeyValueIterators.emptyIterator()); @@ -131,6 +151,16 @@ public void shouldDelegateToUnderlyingStoreWhenFindingSessions() { EasyMock.verify(inner); } + @Test + public void shouldDelegateToUnderlyingStoreWhenBackwardFindingSessions() { + EasyMock.expect(inner.backwardFindSessions(bytesKey, 0, 1)).andReturn(KeyValueIterators.emptyIterator()); + + init(); + + store.backwardFindSessions(bytesKey, 0, 1); + EasyMock.verify(inner); + } + @Test public void shouldDelegateToUnderlyingStoreWhenFindingSessionRange() { EasyMock.expect(inner.findSessions(bytesKey, bytesKey, 0, 1)).andReturn(KeyValueIterators.emptyIterator()); @@ -141,6 +171,16 @@ public void shouldDelegateToUnderlyingStoreWhenFindingSessionRange() { EasyMock.verify(inner); } + @Test + public void shouldDelegateToUnderlyingStoreWhenBackwardFindingSessionRange() { + EasyMock.expect(inner.backwardFindSessions(bytesKey, bytesKey, 0, 1)).andReturn(KeyValueIterators.emptyIterator()); + + init(); + + store.backwardFindSessions(bytesKey, bytesKey, 0, 1); + EasyMock.verify(inner); + } + @Test public void shouldFlushUnderlyingStore() { inner.flush(); @@ -162,6 +202,4 @@ public void shouldCloseUnderlyingStore() { store.close(); EasyMock.verify(inner); } - - } \ No newline at end of file diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java index d88cbd0011866..aeef8ce8e13f5 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java @@ -187,11 +187,7 @@ public void shouldGetSegmentsWithinTimeRange() { segments.getOrCreateSegmentIfLive(3, context, streamTime); segments.getOrCreateSegmentIfLive(4, context, streamTime); -<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); -======= - final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); ->>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(0, segments.get(0).id); assertEquals(1, segments.get(1).id); @@ -211,11 +207,7 @@ public void shouldGetSegmentsWithinBackwardTimeRange() { segments.getOrCreateSegmentIfLive(3, context, streamTime); segments.getOrCreateSegmentIfLive(4, context, streamTime); -<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); -======= - final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); ->>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(0, segments.get(2).id); assertEquals(1, segments.get(1).id); @@ -230,11 +222,7 @@ public void shouldGetSegmentsWithinTimeRangeOutOfOrder() { updateStreamTimeAndCreateSegment(1); updateStreamTimeAndCreateSegment(3); -<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); -======= - final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); ->>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(0, segments.get(0).id); assertEquals(1, segments.get(1).id); @@ -249,11 +237,7 @@ public void shouldGetSegmentsWithinTimeBackwardRangeOutOfOrder() { updateStreamTimeAndCreateSegment(1); updateStreamTimeAndCreateSegment(3); -<<<<<<< HEAD final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, false); -======= - final List segments = this.segments.segments(0, 2 * SEGMENT_INTERVAL, true); ->>>>>>> e431f38ea... key/value reverse operation assertEquals(3, segments.size()); assertEquals(2, segments.get(0).id); assertEquals(1, segments.get(1).id); @@ -360,11 +344,7 @@ public void shouldClearSegmentsOnClose() { } private void verifyCorrectSegments(final long first, final int numSegments) { -<<<<<<< HEAD final List result = this.segments.segments(0, Long.MAX_VALUE, true); -======= - final List result = this.segments.segments(0, Long.MAX_VALUE, false); ->>>>>>> e431f38ea... key/value reverse operation assertEquals(numSegments, result.size()); for (int i = 0; i < numSegments; i++) { assertEquals(i + first, result.get(i).id); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java index 617ff362b694a..219009a13dfb3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java @@ -45,7 +45,7 @@ public long segmentId(final Bytes key) { private final SessionWindow storeWindow = new SessionWindow(0, 1); private final Iterator, byte[]>> storeKvs = Collections.singleton( - KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get())).iterator(); + KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get())).iterator(); private final SessionWindow cacheWindow = new SessionWindow(10, 20); private final Iterator> cacheKvs = Collections.singleton( KeyValue.pair( @@ -55,56 +55,101 @@ public long segmentId(final Bytes key) { @Test public void shouldHaveNextFromStore() { - final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator()); + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator(), false); + assertTrue(mergeIterator.hasNext()); + } + + @Test + public void shouldHaveNextFromReverseStore() { + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator(), true); assertTrue(mergeIterator.hasNext()); } @Test public void shouldGetNextFromStore() { - final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator()); + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator(), false); + assertThat(mergeIterator.next(), equalTo(KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get()))); + } + + @Test + public void shouldGetNextFromReverseStore() { + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator(), true); assertThat(mergeIterator.next(), equalTo(KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get()))); } @Test public void shouldPeekNextKeyFromStore() { - final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator()); + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator(), false); + assertThat(mergeIterator.peekNextKey(), equalTo(new Windowed<>(storeKey, storeWindow))); + } + + @Test + public void shouldPeekNextKeyFromReverseStore() { + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(storeKvs, Collections.emptyIterator(), true); assertThat(mergeIterator.peekNextKey(), equalTo(new Windowed<>(storeKey, storeWindow))); } @Test public void shouldHaveNextFromCache() { - final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs); + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs, false); + assertTrue(mergeIterator.hasNext()); + } + + @Test + public void shouldHaveNextFromReverseCache() { + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs, true); assertTrue(mergeIterator.hasNext()); } @Test public void shouldGetNextFromCache() { - final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs); + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs, false); + assertThat(mergeIterator.next(), equalTo(KeyValue.pair(new Windowed<>(cacheKey, cacheWindow), cacheKey.get()))); + } + + @Test + public void shouldGetNextFromReverseCache() { + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs, true); assertThat(mergeIterator.next(), equalTo(KeyValue.pair(new Windowed<>(cacheKey, cacheWindow), cacheKey.get()))); } @Test public void shouldPeekNextKeyFromCache() { - final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs); + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs, false); + assertThat(mergeIterator.peekNextKey(), equalTo(new Windowed<>(cacheKey, cacheWindow))); + } + + @Test + public void shouldPeekNextKeyFromReverseCache() { + final MergedSortedCacheSessionStoreIterator mergeIterator = createIterator(Collections.emptyIterator(), cacheKvs, true); assertThat(mergeIterator.peekNextKey(), equalTo(new Windowed<>(cacheKey, cacheWindow))); } @Test public void shouldIterateBothStoreAndCache() { - final MergedSortedCacheSessionStoreIterator iterator = createIterator(storeKvs, cacheKvs); + final MergedSortedCacheSessionStoreIterator iterator = createIterator(storeKvs, cacheKvs, false); assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get()))); assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(cacheKey, cacheWindow), cacheKey.get()))); assertFalse(iterator.hasNext()); } + @Test + public void shouldReverseIterateBothStoreAndCache() { + final MergedSortedCacheSessionStoreIterator iterator = createIterator(storeKvs, cacheKvs, true); + assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(cacheKey, cacheWindow), cacheKey.get()))); + assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get()))); + assertFalse(iterator.hasNext()); + } + private MergedSortedCacheSessionStoreIterator createIterator(final Iterator, byte[]>> storeKvs, - final Iterator> cacheKvs) { + final Iterator> cacheKvs, + final boolean reverse) { final DelegatingPeekingKeyValueIterator, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>("store", new KeyValueIteratorStub<>(storeKvs)); final PeekingKeyValueIterator cacheIterator = new DelegatingPeekingKeyValueIterator<>("cache", new KeyValueIteratorStub<>(cacheKvs)); - return new MergedSortedCacheSessionStoreIterator(cacheIterator, storeIterator, SINGLE_SEGMENT_CACHE_FUNCTION); + return new MergedSortedCacheSessionStoreIterator(cacheIterator, storeIterator, SINGLE_SEGMENT_CACHE_FUNCTION, reverse); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java index 0ff822ebb697c..5f48c90c36c1c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java @@ -250,8 +250,8 @@ public void shouldWriteBytesToInnerStoreAndRecordPutMetric() { @Test public void shouldFindSessionsFromStoreAndRecordFetchMetric() { expect(innerStore.findSessions(KEY_BYTES, 0, 0)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.findSessions(KEY, 0, 0); @@ -264,11 +264,28 @@ public void shouldFindSessionsFromStoreAndRecordFetchMetric() { verify(innerStore); } + @Test + public void shouldBackwardFindSessionsFromStoreAndRecordFetchMetric() { + expect(innerStore.backwardFindSessions(KEY_BYTES, 0, 0)) + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + init(); + + final KeyValueIterator, String> iterator = store.backwardFindSessions(KEY, 0, 0); + assertThat(iterator.next().value, equalTo(VALUE)); + assertFalse(iterator.hasNext()); + iterator.close(); + + final KafkaMetric metric = metric("fetch-rate"); + assertTrue((Double) metric.metricValue() > 0); + verify(innerStore); + } + @Test public void shouldFindSessionRangeFromStoreAndRecordFetchMetric() { expect(innerStore.findSessions(KEY_BYTES, KEY_BYTES, 0, 0)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.findSessions(KEY, KEY, 0, 0); @@ -281,6 +298,23 @@ public void shouldFindSessionRangeFromStoreAndRecordFetchMetric() { verify(innerStore); } + @Test + public void shouldBackwardFindSessionRangeFromStoreAndRecordFetchMetric() { + expect(innerStore.backwardFindSessions(KEY_BYTES, KEY_BYTES, 0, 0)) + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + init(); + + final KeyValueIterator, String> iterator = store.backwardFindSessions(KEY, KEY, 0, 0); + assertThat(iterator.next().value, equalTo(VALUE)); + assertFalse(iterator.hasNext()); + iterator.close(); + + final KafkaMetric metric = metric("fetch-rate"); + assertTrue((Double) metric.metricValue() > 0); + verify(innerStore); + } + @Test public void shouldRemoveFromStoreAndRecordRemoveMetric() { innerStore.remove(WINDOWED_KEY_BYTES); @@ -298,8 +332,8 @@ public void shouldRemoveFromStoreAndRecordRemoveMetric() { @Test public void shouldFetchForKeyAndRecordFetchMetric() { expect(innerStore.fetch(KEY_BYTES)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.fetch(KEY); @@ -312,11 +346,28 @@ public void shouldFetchForKeyAndRecordFetchMetric() { verify(innerStore); } + @Test + public void shouldBackwardFetchForKeyAndRecordFetchMetric() { + expect(innerStore.backwardFetch(KEY_BYTES)) + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + init(); + + final KeyValueIterator, String> iterator = store.backwardFetch(KEY); + assertThat(iterator.next().value, equalTo(VALUE)); + assertFalse(iterator.hasNext()); + iterator.close(); + + final KafkaMetric metric = metric("fetch-rate"); + assertTrue((Double) metric.metricValue() > 0); + verify(innerStore); + } + @Test public void shouldFetchRangeFromStoreAndRecordFetchMetric() { expect(innerStore.fetch(KEY_BYTES, KEY_BYTES)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.fetch(KEY, KEY); @@ -329,6 +380,23 @@ public void shouldFetchRangeFromStoreAndRecordFetchMetric() { verify(innerStore); } + @Test + public void shouldBackwardFetchRangeFromStoreAndRecordFetchMetric() { + expect(innerStore.backwardFetch(KEY_BYTES, KEY_BYTES)) + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + init(); + + final KeyValueIterator, String> iterator = store.backwardFetch(KEY, KEY); + assertThat(iterator.next().value, equalTo(VALUE)); + assertFalse(iterator.hasNext()); + iterator.close(); + + final KafkaMetric metric = metric("fetch-rate"); + assertTrue((Double) metric.metricValue() > 0); + verify(innerStore); + } + @Test public void shouldRecordRestoreTimeOnInit() { init(); @@ -384,7 +452,8 @@ public void shouldThrowNullPointerOnFindSessionsRangeIfToIsNull() { store.findSessions("a", null, 0, 0); } - private interface CachedSessionStore extends SessionStore, CachedStateStore { } + private interface CachedSessionStore extends SessionStore, CachedStateStore { + } @SuppressWarnings("unchecked") @Test @@ -441,9 +510,9 @@ private KafkaMetric metric(final String name) { private List storeMetrics() { return metrics.metrics() - .keySet() - .stream() - .filter(name -> name.group().equals(storeLevelGroup) && name.tags().equals(tags)) - .collect(Collectors.toList()); + .keySet() + .stream() + .filter(name -> name.group().equals(storeLevelGroup) && name.tags().equals(tags)) + .collect(Collectors.toList()); } } diff --git a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java index 4f6d5debb0ada..81df1337ea76e 100644 --- a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java +++ b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java @@ -32,7 +32,7 @@ import java.util.TreeMap; public class ReadOnlySessionStoreStub implements ReadOnlySessionStore, StateStore { - private NavigableMap, V>>> sessions = new TreeMap<>(); + private final NavigableMap, V>>> sessions = new TreeMap<>(); private boolean open = true; public void put(final Windowed sessionKey, final V value) { @@ -42,6 +42,31 @@ public void put(final Windowed sessionKey, final V value) { sessions.get(sessionKey.key()).add(KeyValue.pair(sessionKey, value)); } + @Override + public KeyValueIterator, V> findSessions(K key, long earliestSessionEndTime, long latestSessionStartTime) { + return null; + } + + @Override + public KeyValueIterator, V> backwardFindSessions(K key, long earliestSessionEndTime, long latestSessionStartTime) { + return null; + } + + @Override + public KeyValueIterator, V> findSessions(K keyFrom, K keyTo, long earliestSessionEndTime, long latestSessionStartTime) { + return null; + } + + @Override + public KeyValueIterator, V> backwardFindSessions(K keyFrom, K keyTo, long earliestSessionEndTime, long latestSessionStartTime) { + return null; + } + + @Override + public V fetchSession(K key, long startTime, long endTime) { + return null; + } + @Override public KeyValueIterator, V> fetch(final K key) { if (!open) { @@ -53,6 +78,17 @@ public KeyValueIterator, V> fetch(final K key) { return new KeyValueIteratorStub<>(sessions.get(key).iterator()); } + @Override + public KeyValueIterator, V> backwardFetch(K key) { + if (!open) { + throw new InvalidStateStoreException("not open"); + } + if (!sessions.containsKey(key)) { + return new KeyValueIteratorStub<>(Collections., V>>emptyIterator()); + } + return new KeyValueIteratorStub<>(sessions.descendingMap().get(key).iterator()); + } + @Override public KeyValueIterator, V> fetch(final K from, final K to) { if (!open) { @@ -61,7 +97,7 @@ public KeyValueIterator, V> fetch(final K from, final K to) { if (sessions.subMap(from, true, to, true).isEmpty()) { return new KeyValueIteratorStub<>(Collections., V>>emptyIterator()); } - final Iterator, V>>> keysIterator = sessions.subMap(from, true, to, true).values().iterator(); + final Iterator, V>>> keysIterator = sessions.subMap(from, true, to, true).values().iterator(); return new KeyValueIteratorStub<>( new Iterator, V>>() { @@ -82,11 +118,15 @@ public boolean hasNext() { public KeyValue, V> next() { return it.next(); } - } ); } + @Override + public KeyValueIterator, V> backwardFetch(K from, K to) { + return null; + } + @Override public String name() { return ""; From 1811b577b027506d6249e1d71690bc82107ef2bf Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 11 Aug 2020 13:28:20 +0100 Subject: [PATCH 04/14] improve time range comments --- .../streams/state/ReadOnlySessionStore.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java index 018dbedb3cabc..5f7654ca201ef 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java @@ -36,9 +36,9 @@ public interface ReadOnlySessionStore { * This iterator must be closed after use. * * @param key the key to return sessions for - * @param earliestSessionEndTime the end timestamp of the earliest session to search for - * @param latestSessionStartTime the end timestamp of the latest session to search for - * @return iterator of sessions with the matching key and aggregated values + * @param earliestSessionEndTime the end timestamp of the earliest session to search for, where iteration starts. + * @param latestSessionStartTime the end timestamp of the latest session to search for, where iteration ends. + * @return iterator of sessions with the matching key and aggregated values, from earliest to latest session time. * @throws NullPointerException If null is used for key. */ default KeyValueIterator, AGG> findSessions(final K key, @@ -54,9 +54,9 @@ default KeyValueIterator, AGG> findSessions(final K key, * This iterator must be closed after use. * * @param key the key to return sessions for - * @param earliestSessionEndTime the end timestamp of the earliest session to search for - * @param latestSessionStartTime the end timestamp of the latest session to search for - * @return backward iterator of sessions with the matching key and aggregated values + * @param earliestSessionEndTime the end timestamp of the earliest session to search for, where iteration ends. + * @param latestSessionStartTime the end timestamp of the latest session to search for, where iteration starts. + * @return backward iterator of sessions with the matching key and aggregated values, from latest to earliest session time. * @throws NullPointerException If null is used for key. */ default KeyValueIterator, AGG> backwardFindSessions(final K key, @@ -73,9 +73,9 @@ default KeyValueIterator, AGG> backwardFindSessions(final K key, * * @param keyFrom The first key that could be in the range * @param keyTo The last key that could be in the range - * @param earliestSessionEndTime the end timestamp of the earliest session to search for - * @param latestSessionStartTime the end timestamp of the latest session to search for - * @return iterator of sessions with the matching keys and aggregated values + * @param earliestSessionEndTime the end timestamp of the earliest session to search for, where iteration starts. + * @param latestSessionStartTime the end timestamp of the latest session to search for, where iteration ends. + * @return iterator of sessions with the matching keys and aggregated values, from earliest to latest session time. * @throws NullPointerException If null is used for any key. */ default KeyValueIterator, AGG> findSessions(final K keyFrom, @@ -94,9 +94,9 @@ default KeyValueIterator, AGG> findSessions(final K keyFrom, * * @param keyFrom The first key that could be in the range * @param keyTo The last key that could be in the range - * @param earliestSessionEndTime the end timestamp of the earliest session to search for - * @param latestSessionStartTime the end timestamp of the latest session to search for - * @return backward iterator of sessions with the matching keys and aggregated values + * @param earliestSessionEndTime the end timestamp of the earliest session to search for, where iteration ends. + * @param latestSessionStartTime the end timestamp of the latest session to search for, where iteration starts. + * @return backward iterator of sessions with the matching keys and aggregated values, from latest to earliest session time. * @throws NullPointerException If null is used for any key. */ default KeyValueIterator, AGG> backwardFindSessions(final K keyFrom, @@ -127,7 +127,7 @@ default AGG fetchSession(final K key, final long startTime, final long endTime) * available session to the newest/latest session. * * @param key record key to find aggregated session values for - * @return KeyValueIterator containing all sessions for the provided key. + * @return KeyValueIterator containing all sessions for the provided key, from oldest to newest session. * @throws NullPointerException If null is used for key. */ KeyValueIterator, AGG> fetch(final K key); @@ -140,7 +140,7 @@ default AGG fetchSession(final K key, final long startTime, final long endTime) * available session to the oldest/earliest session. * * @param key record key to find aggregated session values for - * @return backward KeyValueIterator containing all sessions for the provided key. + * @return backward KeyValueIterator containing all sessions for the provided key, from newest to oldest session. * @throws NullPointerException If null is used for key. */ default KeyValueIterator, AGG> backwardFetch(final K key) { @@ -156,7 +156,7 @@ default KeyValueIterator, AGG> backwardFetch(final K key) { * * @param from first key in the range to find aggregated session values for * @param to last key in the range to find aggregated session values for - * @return KeyValueIterator containing all sessions for the provided key. + * @return KeyValueIterator containing all sessions for the provided key, from oldest to newest session. * @throws NullPointerException If null is used for any of the keys. */ KeyValueIterator, AGG> fetch(final K from, final K to); @@ -170,7 +170,7 @@ default KeyValueIterator, AGG> backwardFetch(final K key) { * * @param from first key in the range to find aggregated session values for * @param to last key in the range to find aggregated session values for - * @return backward KeyValueIterator containing all sessions for the provided key. + * @return backward KeyValueIterator containing all sessions for the provided key, from newest to oldest session. * @throws NullPointerException If null is used for any of the keys. */ default KeyValueIterator, AGG> backwardFetch(final K from, final K to) { From 8c26756dac79c2e5f1574c8057aeef3e163c71ea Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 11 Aug 2020 14:07:37 +0100 Subject: [PATCH 05/14] fix bytes range validator not needed --- .../internals/CachingSessionStoreTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java index 7debda23c9b77..53940180eb0f1 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java @@ -591,6 +591,26 @@ public void shouldThrowNullPointerExceptionOnPutNullKey() { assertThrows(NullPointerException.class, () -> cachingStore.put(null, "1".getBytes())); } + @Test + public void shouldNotThrowInvalidBackwardRangeExceptionWithNegativeFromKey() { + final Bytes keyFrom = Bytes.wrap(Serdes.Integer().serializer().serialize("", -1)); + final Bytes keyTo = Bytes.wrap(Serdes.Integer().serializer().serialize("", 1)); + + try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(CachingSessionStore.class)) { + final KeyValueIterator, byte[]> iterator = cachingStore.backwardFindSessions(keyFrom, keyTo, 0L, 10L); + assertFalse(iterator.hasNext()); + + final List messages = appender.getMessages(); + assertThat( + messages, + hasItem("Returning empty iterator for fetch with invalid key range: from > to." + + " This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + + " Note that the built-in numerical serdes do not follow this for negative numbers") + ); + } + } + @Test public void shouldNotThrowInvalidRangeExceptionWithNegativeFromKey() { final Bytes keyFrom = Bytes.wrap(Serdes.Integer().serializer().serialize("", -1)); From e0d160bc3ebf65a02fd995ea59e191a8db90b171 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 11 Aug 2020 14:33:22 +0100 Subject: [PATCH 06/14] fix cache iterator --- .../state/internals/CachingSessionStore.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java index a46777d0b6c76..dc3a998758c2f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java @@ -67,7 +67,6 @@ public void init(final ProcessorContext context, final StateStore root) { super.init(context, root); } - @SuppressWarnings("unchecked") private void initInternal(final InternalProcessorContext context) { this.context = context; @@ -150,7 +149,7 @@ public KeyValueIterator, byte[]> findSessions(final Bytes key, validateStoreOpen(); final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? - new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime) : + new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, false) : context.cache().range(cacheName, cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) @@ -172,7 +171,7 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte validateStoreOpen(); final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? - new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime) : + new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, true) : context.cache().reverseRange(cacheName, cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) @@ -325,8 +324,9 @@ private class CacheIteratorWrapper implements PeekingKeyValueIterator Date: Mon, 7 Sep 2020 15:23:54 +0100 Subject: [PATCH 07/14] fix session store to support backward queries --- .../state/internals/CachingSessionStore.java | 32 +++++++++++-------- .../ChangeLoggingWindowBytesStore.java | 10 ------ .../internals/CachingSessionStoreTest.java | 2 +- .../ChangeLoggingWindowBytesStoreTest.java | 2 -- ...dCacheWrappedSessionStoreIteratorTest.java | 8 ++--- 5 files changed, 24 insertions(+), 30 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java index dc3a998758c2f..c9bb97983ac79 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java @@ -149,7 +149,7 @@ public KeyValueIterator, byte[]> findSessions(final Bytes key, validateStoreOpen(); final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? - new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, false) : + new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, true) : context.cache().range(cacheName, cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) @@ -161,7 +161,7 @@ public KeyValueIterator, byte[]> findSessions(final Bytes key, keySchema.hasNextCondition(key, key, earliestSessionEndTime, latestSessionStartTime); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); - return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, false); + return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, true); } @Override @@ -171,7 +171,7 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte validateStoreOpen(); final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? - new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, true) : + new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, false) : context.cache().reverseRange(cacheName, cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) @@ -186,7 +186,7 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte latestSessionStartTime); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); - return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, true); + return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, false); } @Override @@ -324,7 +324,7 @@ private class CacheIteratorWrapper implements PeekingKeyValueIterator, byte[]> backwardFetch(final Bytes keyFr return wrapped().backwardFetch(keyFrom, keyTo, timeFrom, timeTo); } - @Override - public KeyValueIterator, byte[]> backwardFetch(final Bytes keyFrom, - final Bytes keyTo, - final Instant from, - final Instant to) { - return wrapped().backwardFetch(keyFrom, keyTo, from, to); - } - @Override public KeyValueIterator, byte[]> all() { return wrapped().all(); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java index 53940180eb0f1..d939686708c14 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java @@ -199,7 +199,7 @@ private void setUpCloseTests() { EasyMock.replay(underlyingStore); cachingStore = new CachingSessionStore(underlyingStore, SEGMENT_INTERVAL); cache = EasyMock.niceMock(ThreadCache.class); - InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); + final InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); context.setRecordContext(new ProcessorRecordContext(10, 0, 0, TOPIC, null)); cachingStore.init((StateStoreContext) context, cachingStore); } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java index c4981ab0f40be..c877ac64f5ab1 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java @@ -32,8 +32,6 @@ import org.junit.Test; import org.junit.runner.RunWith; -import java.time.Instant; - import static java.time.Instant.ofEpochMilli; @RunWith(EasyMockRunner.class) diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java index 219009a13dfb3..d7adca45fb989 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java @@ -127,7 +127,7 @@ public void shouldPeekNextKeyFromReverseCache() { @Test public void shouldIterateBothStoreAndCache() { - final MergedSortedCacheSessionStoreIterator iterator = createIterator(storeKvs, cacheKvs, false); + final MergedSortedCacheSessionStoreIterator iterator = createIterator(storeKvs, cacheKvs, true); assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get()))); assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(cacheKey, cacheWindow), cacheKey.get()))); assertFalse(iterator.hasNext()); @@ -135,7 +135,7 @@ public void shouldIterateBothStoreAndCache() { @Test public void shouldReverseIterateBothStoreAndCache() { - final MergedSortedCacheSessionStoreIterator iterator = createIterator(storeKvs, cacheKvs, true); + final MergedSortedCacheSessionStoreIterator iterator = createIterator(storeKvs, cacheKvs, false); assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(cacheKey, cacheWindow), cacheKey.get()))); assertThat(iterator.next(), equalTo(KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get()))); assertFalse(iterator.hasNext()); @@ -143,13 +143,13 @@ public void shouldReverseIterateBothStoreAndCache() { private MergedSortedCacheSessionStoreIterator createIterator(final Iterator, byte[]>> storeKvs, final Iterator> cacheKvs, - final boolean reverse) { + final boolean forward) { final DelegatingPeekingKeyValueIterator, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>("store", new KeyValueIteratorStub<>(storeKvs)); final PeekingKeyValueIterator cacheIterator = new DelegatingPeekingKeyValueIterator<>("cache", new KeyValueIteratorStub<>(cacheKvs)); - return new MergedSortedCacheSessionStoreIterator(cacheIterator, storeIterator, SINGLE_SEGMENT_CACHE_FUNCTION, reverse); + return new MergedSortedCacheSessionStoreIterator(cacheIterator, storeIterator, SINGLE_SEGMENT_CACHE_FUNCTION, forward); } } From 64a95c01658f1281e6e7b8986b9678dcbf456ffc Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 6 Oct 2020 18:04:59 +0100 Subject: [PATCH 08/14] improve format based on feedback --- .../state/internals/CachingSessionStore.java | 33 +++++++++++++------ .../state/internals/InMemorySessionStore.java | 20 ++++++----- .../AbstractSessionBytesStoreTest.java | 4 +-- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java index c9bb97983ac79..31589aa03f533 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java @@ -420,22 +420,35 @@ private long currentSegmentLastTime() { } private void getNextSegmentIterator() { - ++currentSegmentId; - lastSegmentId = cacheFunction.segmentId(maxObservedTimestamp); + if (forward) { + ++currentSegmentId; + lastSegmentId = cacheFunction.segmentId(maxObservedTimestamp); - if (currentSegmentId > lastSegmentId) { - current = null; - return; - } + if (currentSegmentId > lastSegmentId) { + current = null; + return; + } - setCacheKeyRange(currentSegmentBeginTime(), currentSegmentLastTime()); + setCacheKeyRange(currentSegmentBeginTime(), currentSegmentLastTime()); + + current.close(); - current.close(); - if (forward) { current = context.cache().range(cacheName, cacheKeyFrom, cacheKeyTo); } else { + --currentSegmentId; + + if (currentSegmentId < lastSegmentId) { + current = null; + return; + } + + setCacheKeyRange(currentSegmentBeginTime(), currentSegmentLastTime()); + + current.close(); + current = context.cache().reverseRange(cacheName, cacheKeyFrom, cacheKeyTo); } + } private void setCacheKeyRange(final long lowerRangeEndTime, final long upperRangeEndTime) { @@ -443,7 +456,7 @@ private void setCacheKeyRange(final long lowerRangeEndTime, final long upperRang throw new IllegalStateException("Error iterating over segments: segment interval has changed"); } - if (keyFrom == keyTo) { + if (keyFrom.equals(keyTo)) { cacheKeyFrom = cacheFunction.cacheKey(segmentLowerRangeFixedSize(keyFrom, lowerRangeEndTime)); cacheKeyTo = cacheFunction.cacheKey(segmentUpperRangeFixedSize(keyTo, upperRangeEndTime)); } else { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java index f7bea499d20a7..3aad2d1e0458d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java @@ -250,7 +250,8 @@ public KeyValueIterator, byte[]> fetch(final Bytes key) { return registerNewIterator( key, key, - Long.MAX_VALUE, endTimeMap.entrySet().iterator(), + Long.MAX_VALUE, + endTimeMap.entrySet().iterator(), true); } @@ -484,13 +485,16 @@ private boolean setInnerIterators() { recordIterator = entries.iterator(); } else { final Set> entries; - if (forward) entries = nextKeyEntry.getValue() - .headMap(latestSessionStartTime, true) - .descendingMap() - .entrySet(); - else entries = nextKeyEntry.getValue() - .headMap(latestSessionStartTime, true) - .entrySet(); + if (forward) { + entries = nextKeyEntry.getValue() + .headMap(latestSessionStartTime, true) + .descendingMap() + .entrySet(); + } else { + entries = nextKeyEntry.getValue() + .headMap(latestSessionStartTime, true) + .entrySet(); + } recordIterator = entries.iterator(); } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java index 795515e7de261..8130e02e66523 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java @@ -405,7 +405,7 @@ public void shouldFetchExactKeys() { @Test public void shouldBackwardFetchExactKeys() { sessionStore = buildSessionStore(0x7a00000000000000L, Serdes.String(), Serdes.Long()); - sessionStore.init(context, sessionStore); + sessionStore.init((StateStoreContext) context, sessionStore); sessionStore.put(new Windowed<>("a", new SessionWindow(0, 0)), 1L); sessionStore.put(new Windowed<>("aa", new SessionWindow(0, 10)), 2L); @@ -473,7 +473,7 @@ public void shouldBackwardFetchAndIterateOverExactBinaryKeys() { final SessionStore sessionStore = buildSessionStore(RETENTION_PERIOD, Serdes.Bytes(), Serdes.String()); - sessionStore.init(context, sessionStore); + sessionStore.init((StateStoreContext) context, sessionStore); final Bytes key1 = Bytes.wrap(new byte[] {0}); final Bytes key2 = Bytes.wrap(new byte[] {0, 0}); From a875cf1237a75eed6ccf3e00321768623aa8af11 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 6 Oct 2020 19:06:13 +0100 Subject: [PATCH 09/14] test caching store across segments --- .../internals/CachingSessionStoreTest.java | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java index d939686708c14..7a7634583695b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java @@ -316,8 +316,8 @@ public void shouldFetchCorrectlyAcrossSegments() { final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); cachingStore.put(a1, "1".getBytes()); cachingStore.put(a2, "2".getBytes()); - cachingStore.put(a3, "3".getBytes()); cachingStore.flush(); + cachingStore.put(a3, "3".getBytes()); final KeyValueIterator, byte[]> results = cachingStore.findSessions(keyA, 0, SEGMENT_INTERVAL * 2); assertEquals(a1, results.next().key); @@ -326,6 +326,23 @@ public void shouldFetchCorrectlyAcrossSegments() { assertFalse(results.hasNext()); } + @Test + public void shouldBackwardFetchCorrectlyAcrossSegments() { + final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); + final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + cachingStore.put(a1, "1".getBytes()); + cachingStore.put(a2, "2".getBytes()); + cachingStore.flush(); + cachingStore.put(a3, "3".getBytes()); + final KeyValueIterator, byte[]> results = + cachingStore.backwardFindSessions(keyA, 0, SEGMENT_INTERVAL * 2); + assertEquals(a3, results.next().key); + assertEquals(a2, results.next().key); + assertEquals(a1, results.next().key); + assertFalse(results.hasNext()); + } + @Test public void shouldFetchRangeCorrectlyAcrossSegments() { final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); @@ -518,6 +535,24 @@ public void shouldReturnSameResultsForSingleKeyFindSessionsAndEqualKeyRangeFindS assertFalse(keyRangeIterator.hasNext()); } + @Test + public void shouldReturnSameResultsForSingleKeyFindSessionsBackwardsAndEqualKeyRangeFindSessions() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 1)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(2, 3)), "2".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(4, 5)), "3".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(6, 7)), "4".getBytes()); + + final KeyValueIterator, byte[]> singleKeyIterator = + cachingStore.backwardFindSessions(keyAA, 0L, 10L); + final KeyValueIterator, byte[]> keyRangeIterator = + cachingStore.backwardFindSessions(keyAA, keyAA, 0L, 10L); + + assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); + assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); + assertFalse(singleKeyIterator.hasNext()); + assertFalse(keyRangeIterator.hasNext()); + } + @Test public void shouldClearNamespaceCacheOnClose() { final Windowed a1 = new Windowed<>(keyA, new SessionWindow(0, 0)); From 76bb9a27fd7a5e58bcfb74c8eca70c295f573d20 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Wed, 7 Oct 2020 10:40:10 +0100 Subject: [PATCH 10/14] improve testing names and scenarios --- .../internals/CachingSessionStoreTest.java | 40 ++++++++++++------ .../kafka/test/ReadOnlySessionStoreStub.java | 41 ++++++++++++++++--- 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java index 7a7634583695b..0da96bc3c6515 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java @@ -44,15 +44,13 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Random; -import java.util.Set; import static java.util.Arrays.asList; -import static org.apache.kafka.common.utils.Utils.mkSet; import static org.apache.kafka.test.StreamsTestUtils.toList; import static org.apache.kafka.test.StreamsTestUtils.verifyKeyValueList; import static org.apache.kafka.test.StreamsTestUtils.verifyWindowedKeyValue; @@ -314,15 +312,24 @@ public void shouldFetchCorrectlyAcrossSegments() { final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + final Windowed a4 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 3, SEGMENT_INTERVAL * 3)); + final Windowed a5 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 4, SEGMENT_INTERVAL * 4)); + final Windowed a6 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 5, SEGMENT_INTERVAL * 5)); cachingStore.put(a1, "1".getBytes()); cachingStore.put(a2, "2".getBytes()); - cachingStore.flush(); cachingStore.put(a3, "3".getBytes()); + cachingStore.flush(); + cachingStore.put(a4, "4".getBytes()); + cachingStore.put(a5, "5".getBytes()); + cachingStore.put(a6, "6".getBytes()); final KeyValueIterator, byte[]> results = - cachingStore.findSessions(keyA, 0, SEGMENT_INTERVAL * 2); + cachingStore.findSessions(keyA, 0, SEGMENT_INTERVAL * 5); assertEquals(a1, results.next().key); assertEquals(a2, results.next().key); assertEquals(a3, results.next().key); + assertEquals(a4, results.next().key); + assertEquals(a5, results.next().key); + assertEquals(a6, results.next().key); assertFalse(results.hasNext()); } @@ -331,12 +338,21 @@ public void shouldBackwardFetchCorrectlyAcrossSegments() { final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + final Windowed a4 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 3, SEGMENT_INTERVAL * 3)); + final Windowed a5 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 4, SEGMENT_INTERVAL * 4)); + final Windowed a6 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 5, SEGMENT_INTERVAL * 5)); cachingStore.put(a1, "1".getBytes()); cachingStore.put(a2, "2".getBytes()); - cachingStore.flush(); cachingStore.put(a3, "3".getBytes()); + cachingStore.flush(); + cachingStore.put(a4, "4".getBytes()); + cachingStore.put(a5, "5".getBytes()); + cachingStore.put(a6, "6".getBytes()); final KeyValueIterator, byte[]> results = - cachingStore.backwardFindSessions(keyA, 0, SEGMENT_INTERVAL * 2); + cachingStore.backwardFindSessions(keyA, 0, SEGMENT_INTERVAL * 5); + assertEquals(a6, results.next().key); + assertEquals(a5, results.next().key); + assertEquals(a4, results.next().key); assertEquals(a3, results.next().key); assertEquals(a2, results.next().key); assertEquals(a1, results.next().key); @@ -358,12 +374,12 @@ public void shouldFetchRangeCorrectlyAcrossSegments() { final KeyValueIterator, byte[]> rangeResults = cachingStore.findSessions(keyA, keyAA, 0, SEGMENT_INTERVAL * 2); - final Set> keys = new HashSet<>(); + final List> keys = new ArrayList<>(); while (rangeResults.hasNext()) { keys.add(rangeResults.next().key); } rangeResults.close(); - assertEquals(mkSet(a1, a2, a3, aa1, aa3), keys); + assertEquals(Arrays.asList(a1, aa1, a2, a3, aa3), keys); } @Test @@ -381,12 +397,12 @@ public void shouldBackwardFetchRangeCorrectlyAcrossSegments() { final KeyValueIterator, byte[]> rangeResults = cachingStore.backwardFindSessions(keyA, keyAA, 0, SEGMENT_INTERVAL * 2); - final Set> keys = new HashSet<>(); + final List> keys = new ArrayList<>(); while (rangeResults.hasNext()) { keys.add(rangeResults.next().key); } rangeResults.close(); - assertEquals(mkSet(a1, a2, a3, aa1, aa3), keys); + assertEquals(Arrays.asList(aa3, a3, a2, aa1, a1), keys); } @Test @@ -627,7 +643,7 @@ public void shouldThrowNullPointerExceptionOnPutNullKey() { } @Test - public void shouldNotThrowInvalidBackwardRangeExceptionWithNegativeFromKey() { + public void shouldNotThrowInvalidRangeExceptionWhenBackwardWithNegativeFromKey() { final Bytes keyFrom = Bytes.wrap(Serdes.Integer().serializer().serialize("", -1)); final Bytes keyTo = Bytes.wrap(Serdes.Integer().serializer().serialize("", 1)); diff --git a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java index 81df1337ea76e..0c8b27630c0da 100644 --- a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java +++ b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java @@ -44,27 +44,27 @@ public void put(final Windowed sessionKey, final V value) { @Override public KeyValueIterator, V> findSessions(K key, long earliestSessionEndTime, long latestSessionStartTime) { - return null; + throw new UnsupportedOperationException("Moved from Session Store. Implement if needed"); } @Override public KeyValueIterator, V> backwardFindSessions(K key, long earliestSessionEndTime, long latestSessionStartTime) { - return null; + throw new UnsupportedOperationException("Moved from Session Store. Implement if needed"); } @Override public KeyValueIterator, V> findSessions(K keyFrom, K keyTo, long earliestSessionEndTime, long latestSessionStartTime) { - return null; + throw new UnsupportedOperationException("Moved from Session Store. Implement if needed"); } @Override public KeyValueIterator, V> backwardFindSessions(K keyFrom, K keyTo, long earliestSessionEndTime, long latestSessionStartTime) { - return null; + throw new UnsupportedOperationException("Moved from Session Store. Implement if needed"); } @Override public V fetchSession(K key, long startTime, long endTime) { - return null; + throw new UnsupportedOperationException("Moved from Session Store. Implement if needed"); } @Override @@ -124,7 +124,36 @@ public KeyValue, V> next() { @Override public KeyValueIterator, V> backwardFetch(K from, K to) { - return null; + if (!open) { + throw new InvalidStateStoreException("not open"); + } + if (sessions.subMap(from, true, to, true).isEmpty()) { + return new KeyValueIteratorStub<>(Collections., V>>emptyIterator()); + } + final Iterator, V>>> keysIterator = sessions.subMap(from, true, to, true) + .descendingMap().values().iterator(); + return new KeyValueIteratorStub<>( + new Iterator, V>>() { + + Iterator, V>> it; + + @Override + public boolean hasNext() { + while (it == null || !it.hasNext()) { + if (!keysIterator.hasNext()) { + return false; + } + it = keysIterator.next().iterator(); + } + return true; + } + + @Override + public KeyValue, V> next() { + return it.next(); + } + } + ); } @Override From e49f258ad78bb75f23481f030d5e7be1db12ce00 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Wed, 7 Oct 2020 23:00:02 +0100 Subject: [PATCH 11/14] expand caching store tests per backing store --- .../state/internals/CachingSessionStore.java | 12 +- .../internals/CacheFlushListenerStub.java | 47 ++ ... => CachingInMemoryKeyValueStoreTest.java} | 29 +- ...a => CachingInMemorySessionStoreTest.java} | 6 +- .../CachingPersistentSessionStoreTest.java | 734 ++++++++++++++++++ ... => CachingPersistentWindowStoreTest.java} | 6 +- 6 files changed, 796 insertions(+), 38 deletions(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java rename streams/src/test/java/org/apache/kafka/streams/state/internals/{CachingKeyValueStoreTest.java => CachingInMemoryKeyValueStoreTest.java} (93%) rename streams/src/test/java/org/apache/kafka/streams/state/internals/{CachingSessionStoreTest.java => CachingInMemorySessionStoreTest.java} (99%) create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java rename streams/src/test/java/org/apache/kafka/streams/state/internals/{CachingWindowStoreTest.java => CachingPersistentWindowStoreTest.java} (99%) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java index 31589aa03f533..2a7ca838c0a0b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java @@ -348,17 +348,21 @@ private CacheIteratorWrapper(final Bytes keyFrom, this.keyFrom = keyFrom; this.keyTo = keyTo; this.latestSessionStartTime = latestSessionStartTime; - this.lastSegmentId = cacheFunction.segmentId(maxObservedTimestamp); this.segmentInterval = cacheFunction.getSegmentInterval(); this.forward = forward; - this.currentSegmentId = cacheFunction.segmentId(earliestSessionEndTime); - - setCacheKeyRange(earliestSessionEndTime, currentSegmentLastTime()); if (forward) { + this.currentSegmentId = cacheFunction.segmentId(earliestSessionEndTime); + this.lastSegmentId = cacheFunction.segmentId(maxObservedTimestamp); + + setCacheKeyRange(earliestSessionEndTime, currentSegmentLastTime()); this.current = context.cache().range(cacheName, cacheKeyFrom, cacheKeyTo); } else { + this.lastSegmentId = cacheFunction.segmentId(earliestSessionEndTime); + this.currentSegmentId = cacheFunction.segmentId(maxObservedTimestamp); + + setCacheKeyRange(currentSegmentBeginTime(), Math.min(latestSessionStartTime, maxObservedTimestamp)); this.current = context.cache().reverseRange(cacheName, cacheKeyFrom, cacheKeyTo); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java new file mode 100644 index 0000000000000..11b6300f743cb --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.state.internals; + +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.streams.kstream.internals.Change; + +import java.util.HashMap; +import java.util.Map; + +public class CacheFlushListenerStub implements CacheFlushListener { + final Deserializer keyDeserializer; + final Deserializer valueDeserializer; + final Map> forwarded = new HashMap<>(); + + CacheFlushListenerStub(final Deserializer keyDeserializer, + final Deserializer valueDeserializer) { + this.keyDeserializer = keyDeserializer; + this.valueDeserializer = valueDeserializer; + } + + @Override + public void apply(final byte[] key, + final byte[] newValue, + final byte[] oldValue, + final long timestamp) { + forwarded.put( + keyDeserializer.deserialize(null, key), + new Change<>( + valueDeserializer.deserialize(null, newValue), + valueDeserializer.deserialize(null, oldValue))); + } +} diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemoryKeyValueStoreTest.java similarity index 93% rename from streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java rename to streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemoryKeyValueStoreTest.java index 98f0ba654bf02..425b590bb65e6 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemoryKeyValueStoreTest.java @@ -17,14 +17,12 @@ package org.apache.kafka.streams.state.internals; import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.errors.InvalidStateStoreException; -import org.apache.kafka.streams.kstream.internals.Change; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.StateStoreContext; import org.apache.kafka.streams.processor.internals.MockStreamsMetrics; @@ -43,9 +41,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.apache.kafka.streams.state.internals.ThreadCacheTest.memoryCacheEntrySize; import static org.hamcrest.CoreMatchers.equalTo; @@ -57,7 +53,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -public class CachingKeyValueStoreTest extends AbstractKeyValueStoreTest { +public class CachingInMemoryKeyValueStoreTest extends AbstractKeyValueStoreTest { private final static String TOPIC = "topic"; private static final String CACHE_NAMESPACE = "0_0-store-name"; @@ -502,27 +498,4 @@ private int addItemsToCache() { return i; } - public static class CacheFlushListenerStub implements CacheFlushListener { - final Deserializer keyDeserializer; - final Deserializer valueDeserializer; - final Map> forwarded = new HashMap<>(); - - CacheFlushListenerStub(final Deserializer keyDeserializer, - final Deserializer valueDeserializer) { - this.keyDeserializer = keyDeserializer; - this.valueDeserializer = valueDeserializer; - } - - @Override - public void apply(final byte[] key, - final byte[] newValue, - final byte[] oldValue, - final long timestamp) { - forwarded.put( - keyDeserializer.deserialize(null, key), - new Change<>( - valueDeserializer.deserialize(null, newValue), - valueDeserializer.deserialize(null, oldValue))); - } - } } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java similarity index 99% rename from streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java rename to streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java index 0da96bc3c6515..eadbb8b43c33c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java @@ -65,7 +65,7 @@ import static org.junit.Assert.assertTrue; @SuppressWarnings("PointlessArithmeticExpression") -public class CachingSessionStoreTest { +public class CachingInMemorySessionStoreTest { private static final int MAX_CACHE_SIZE_BYTES = 600; private static final Long DEFAULT_TIMESTAMP = 10L; @@ -77,13 +77,13 @@ public class CachingSessionStoreTest { private final Bytes keyAA = Bytes.wrap("aa".getBytes()); private final Bytes keyB = Bytes.wrap("b".getBytes()); - private SessionStore underlyingStore = - new InMemorySessionStore("store-name", Long.MAX_VALUE, "metric-scope"); + private SessionStore underlyingStore; private CachingSessionStore cachingStore; private ThreadCache cache; @Before public void before() { + underlyingStore = new InMemorySessionStore("store-name", Long.MAX_VALUE, "metric-scope"); cachingStore = new CachingSessionStore(underlyingStore, SEGMENT_INTERVAL); cache = new ThreadCache(new LogContext("testCache "), MAX_CACHE_SIZE_BYTES, new MockStreamsMetrics(new Metrics())); final InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java new file mode 100644 index 0000000000000..c85de42d9efd4 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java @@ -0,0 +1,734 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.state.internals; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.KeyValueTimestamp; +import org.apache.kafka.streams.errors.InvalidStateStoreException; +import org.apache.kafka.streams.kstream.SessionWindowedDeserializer; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.kstream.internals.Change; +import org.apache.kafka.streams.kstream.internals.SessionWindow; +import org.apache.kafka.streams.processor.StateStoreContext; +import org.apache.kafka.streams.processor.internals.MockStreamsMetrics; +import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; +import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender; +import org.apache.kafka.streams.state.KeyValueIterator; +import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.test.InternalMockProcessorContext; +import org.apache.kafka.test.TestUtils; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; + +import static java.util.Arrays.asList; +import static org.apache.kafka.test.StreamsTestUtils.toList; +import static org.apache.kafka.test.StreamsTestUtils.verifyKeyValueList; +import static org.apache.kafka.test.StreamsTestUtils.verifyWindowedKeyValue; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +@SuppressWarnings("PointlessArithmeticExpression") +public class CachingPersistentSessionStoreTest { + + private static final int MAX_CACHE_SIZE_BYTES = 600; + private static final Long DEFAULT_TIMESTAMP = 10L; + private static final long SEGMENT_INTERVAL = 100L; + private static final String TOPIC = "topic"; + private static final String CACHE_NAMESPACE = "0_0-store-name"; + + private final Bytes keyA = Bytes.wrap("a".getBytes()); + private final Bytes keyAA = Bytes.wrap("aa".getBytes()); + private final Bytes keyB = Bytes.wrap("b".getBytes()); + + private SessionStore underlyingStore; + private CachingSessionStore cachingStore; + private ThreadCache cache; + + @Before + public void before() { + final RocksDBSegmentedBytesStore segmented = new RocksDBSegmentedBytesStore( + "store-name", + "metric-scope", + Long.MAX_VALUE, + SEGMENT_INTERVAL, + new SessionKeySchema()); + underlyingStore = new RocksDBSessionStore(segmented); + cachingStore = new CachingSessionStore(underlyingStore, SEGMENT_INTERVAL); + cache = new ThreadCache(new LogContext("testCache "), MAX_CACHE_SIZE_BYTES, new MockStreamsMetrics(new Metrics())); + final InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); + context.setRecordContext(new ProcessorRecordContext(DEFAULT_TIMESTAMP, 0, 0, TOPIC, null)); + cachingStore.init((StateStoreContext) context, cachingStore); + } + + @After + public void after() { + cachingStore.close(); + } + + @Test + public void shouldPutFetchFromCache() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(0, 0)), "1".getBytes()); + + assertEquals(3, cache.size()); + + final KeyValueIterator, byte[]> a = cachingStore.findSessions(keyA, 0, 0); + final KeyValueIterator, byte[]> b = cachingStore.findSessions(keyB, 0, 0); + + verifyWindowedKeyValue(a.next(), new Windowed<>(keyA, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(b.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); + assertFalse(a.hasNext()); + assertFalse(b.hasNext()); + } + + @Test + public void shouldPutFetchAllKeysFromCache() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(0, 0)), "1".getBytes()); + + assertEquals(3, cache.size()); + + final KeyValueIterator, byte[]> all = cachingStore.findSessions(keyA, keyB, 0, 0); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyA, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); + assertFalse(all.hasNext()); + } + + @Test + public void shouldPutBackwardFetchAllKeysFromCache() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(0, 0)), "1".getBytes()); + + assertEquals(3, cache.size()); + + final KeyValueIterator, byte[]> all = cachingStore.backwardFindSessions(keyA, keyB, 0, 0); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(all.next(), new Windowed<>(keyA, new SessionWindow(0, 0)), "1"); + assertFalse(all.hasNext()); + } + + @Test + public void shouldCloseWrappedStoreAndCacheAfterErrorDuringCacheFlush() { + setUpCloseTests(); + EasyMock.reset(cache); + cache.flush(CACHE_NAMESPACE); + EasyMock.expectLastCall().andThrow(new RuntimeException("Simulating an error on flush")); + EasyMock.replay(cache); + EasyMock.reset(underlyingStore); + underlyingStore.close(); + EasyMock.replay(underlyingStore); + + assertThrows(RuntimeException.class, cachingStore::close); + EasyMock.verify(cache, underlyingStore); + } + + @Test + public void shouldCloseWrappedStoreAfterErrorDuringCacheClose() { + setUpCloseTests(); + EasyMock.reset(cache); + cache.flush(CACHE_NAMESPACE); + cache.close(CACHE_NAMESPACE); + EasyMock.expectLastCall().andThrow(new RuntimeException("Simulating an error on close")); + EasyMock.replay(cache); + EasyMock.reset(underlyingStore); + underlyingStore.close(); + EasyMock.replay(underlyingStore); + + assertThrows(RuntimeException.class, cachingStore::close); + EasyMock.verify(cache, underlyingStore); + } + + @Test + public void shouldCloseCacheAfterErrorDuringWrappedStoreClose() { + setUpCloseTests(); + EasyMock.reset(cache); + cache.flush(CACHE_NAMESPACE); + cache.close(CACHE_NAMESPACE); + EasyMock.replay(cache); + EasyMock.reset(underlyingStore); + underlyingStore.close(); + EasyMock.expectLastCall().andThrow(new RuntimeException("Simulating an error on close")); + EasyMock.replay(underlyingStore); + + assertThrows(RuntimeException.class, cachingStore::close); + EasyMock.verify(cache, underlyingStore); + } + + private void setUpCloseTests() { + underlyingStore = EasyMock.createNiceMock(SessionStore.class); + EasyMock.expect(underlyingStore.name()).andStubReturn("store-name"); + EasyMock.expect(underlyingStore.isOpen()).andStubReturn(true); + EasyMock.replay(underlyingStore); + cachingStore = new CachingSessionStore(underlyingStore, SEGMENT_INTERVAL); + cache = EasyMock.niceMock(ThreadCache.class); + final InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); + context.setRecordContext(new ProcessorRecordContext(10, 0, 0, TOPIC, null)); + cachingStore.init((StateStoreContext) context, cachingStore); + } + + @Test + public void shouldPutFetchRangeFromCache() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(0, 0)), "1".getBytes()); + + assertEquals(3, cache.size()); + + final KeyValueIterator, byte[]> some = cachingStore.findSessions(keyAA, keyB, 0, 0); + verifyWindowedKeyValue(some.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(some.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); + assertFalse(some.hasNext()); + } + + @Test + public void shouldPutBackwardFetchRangeFromCache() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(0, 0)), "1".getBytes()); + + assertEquals(3, cache.size()); + + final KeyValueIterator, byte[]> some = cachingStore.backwardFindSessions(keyAA, keyB, 0, 0); + verifyWindowedKeyValue(some.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); + verifyWindowedKeyValue(some.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); + assertFalse(some.hasNext()); + } + + @Test + public void shouldFetchAllSessionsWithSameRecordKey() { + final List, byte[]>> expected = asList( + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(10, 10)), "2".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(100, 100)), "3".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(1000, 1000)), "4".getBytes()) + ); + for (final KeyValue, byte[]> kv : expected) { + cachingStore.put(kv.key, kv.value); + } + + // add one that shouldn't appear in the results + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "5".getBytes()); + + final List, byte[]>> results = toList(cachingStore.fetch(keyA)); + verifyKeyValueList(expected, results); + } + + @Test + public void shouldBackwardFetchAllSessionsWithSameRecordKey() { + final List, byte[]>> expected = asList( + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(10, 10)), "2".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(100, 100)), "3".getBytes()), + KeyValue.pair(new Windowed<>(keyA, new SessionWindow(1000, 1000)), "4".getBytes()) + ); + for (final KeyValue, byte[]> kv : expected) { + cachingStore.put(kv.key, kv.value); + } + + // add one that shouldn't appear in the results + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(0, 0)), "5".getBytes()); + + final List, byte[]>> results = toList(cachingStore.backwardFetch(keyA)); + Collections.reverse(results); + verifyKeyValueList(expected, results); + } + + @Test + public void shouldFlushItemsToStoreOnEviction() { + final List, byte[]>> added = addSessionsUntilOverflow("a", "b", "c", "d"); + assertEquals(added.size() - 1, cache.size()); + final KeyValueIterator, byte[]> iterator = cachingStore.findSessions(added.get(0).key.key(), 0, 0); + final KeyValue, byte[]> next = iterator.next(); + assertEquals(added.get(0).key, next.key); + assertArrayEquals(added.get(0).value, next.value); + } + + @Test + public void shouldQueryItemsInCacheAndStore() { + final List, byte[]>> added = addSessionsUntilOverflow("a"); + final KeyValueIterator, byte[]> iterator = cachingStore.findSessions( + Bytes.wrap("a".getBytes(StandardCharsets.UTF_8)), + 0, + added.size() * 10); + final List, byte[]>> actual = toList(iterator); + verifyKeyValueList(added, actual); + } + + @Test + public void shouldRemove() { + final Windowed a = new Windowed<>(keyA, new SessionWindow(0, 0)); + final Windowed b = new Windowed<>(keyB, new SessionWindow(0, 0)); + cachingStore.put(a, "2".getBytes()); + cachingStore.put(b, "2".getBytes()); + cachingStore.remove(a); + + final KeyValueIterator, byte[]> rangeIter = + cachingStore.findSessions(keyA, 0, 0); + assertFalse(rangeIter.hasNext()); + + assertNull(cachingStore.fetchSession(keyA, 0, 0)); + assertThat(cachingStore.fetchSession(keyB, 0, 0), equalTo("2".getBytes())); + + } + + @Test + public void shouldFetchCorrectlyAcrossSegments() { + final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); + final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + final Windowed a4 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 3, SEGMENT_INTERVAL * 3)); + final Windowed a5 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 4, SEGMENT_INTERVAL * 4)); + final Windowed a6 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 5, SEGMENT_INTERVAL * 5)); + cachingStore.put(a1, "1".getBytes()); + cachingStore.put(a2, "2".getBytes()); + cachingStore.put(a3, "3".getBytes()); + cachingStore.flush(); + cachingStore.put(a4, "4".getBytes()); + cachingStore.put(a5, "5".getBytes()); + cachingStore.put(a6, "6".getBytes()); + final KeyValueIterator, byte[]> results = + cachingStore.findSessions(keyA, 0, SEGMENT_INTERVAL * 5); + assertEquals(a1, results.next().key); + assertEquals(a2, results.next().key); + assertEquals(a3, results.next().key); + assertEquals(a4, results.next().key); + assertEquals(a5, results.next().key); + assertEquals(a6, results.next().key); + assertFalse(results.hasNext()); + } + + @Test + public void shouldBackwardFetchCorrectlyAcrossSegments() { + final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); + final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + final Windowed a4 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 3, SEGMENT_INTERVAL * 3)); + final Windowed a5 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 4, SEGMENT_INTERVAL * 4)); + final Windowed a6 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 5, SEGMENT_INTERVAL * 5)); + cachingStore.put(a1, "1".getBytes()); + cachingStore.put(a2, "2".getBytes()); + cachingStore.put(a3, "3".getBytes()); + cachingStore.flush(); + cachingStore.put(a4, "4".getBytes()); + cachingStore.put(a5, "5".getBytes()); + cachingStore.put(a6, "6".getBytes()); + final KeyValueIterator, byte[]> results = + cachingStore.backwardFindSessions(keyA, 0, SEGMENT_INTERVAL * 5); + assertEquals(a6, results.next().key); + assertEquals(a5, results.next().key); + assertEquals(a4, results.next().key); + assertEquals(a3, results.next().key); + assertEquals(a2, results.next().key); + assertEquals(a1, results.next().key); + assertFalse(results.hasNext()); + } + + @Test + public void shouldFetchRangeCorrectlyAcrossSegments() { + final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed aa1 = new Windowed<>(keyAA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); + final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + final Windowed aa3 = new Windowed<>(keyAA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + cachingStore.put(a1, "1".getBytes()); + cachingStore.put(aa1, "1".getBytes()); + cachingStore.put(a2, "2".getBytes()); + cachingStore.put(a3, "3".getBytes()); + cachingStore.put(aa3, "3".getBytes()); + + final KeyValueIterator, byte[]> rangeResults = + cachingStore.findSessions(keyA, keyAA, 0, SEGMENT_INTERVAL * 2); + final List> keys = new ArrayList<>(); + while (rangeResults.hasNext()) { + keys.add(rangeResults.next().key); + } + rangeResults.close(); + assertEquals(Arrays.asList(a1, aa1, a2, a3, aa3), keys); + } + + @Test + public void shouldBackwardFetchRangeCorrectlyAcrossSegments() { + final Windowed a1 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed aa1 = new Windowed<>(keyAA, new SessionWindow(SEGMENT_INTERVAL * 0, SEGMENT_INTERVAL * 0)); + final Windowed a2 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 1, SEGMENT_INTERVAL * 1)); + final Windowed a3 = new Windowed<>(keyA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + final Windowed aa3 = new Windowed<>(keyAA, new SessionWindow(SEGMENT_INTERVAL * 2, SEGMENT_INTERVAL * 2)); + cachingStore.put(a1, "1".getBytes()); + cachingStore.put(aa1, "1".getBytes()); + cachingStore.put(a2, "2".getBytes()); + cachingStore.put(a3, "3".getBytes()); + cachingStore.put(aa3, "3".getBytes()); + + final KeyValueIterator, byte[]> rangeResults = + cachingStore.backwardFindSessions(keyA, keyAA, 0, SEGMENT_INTERVAL * 2); + final List> keys = new ArrayList<>(); + while (rangeResults.hasNext()) { + keys.add(rangeResults.next().key); + } + rangeResults.close(); + assertEquals(Arrays.asList(aa3, a3, a2, aa1, a1), keys); + } + + @Test + public void shouldSetFlushListener() { + assertTrue(cachingStore.setFlushListener(null, true)); + assertTrue(cachingStore.setFlushListener(null, false)); + } + + @Test + public void shouldForwardChangedValuesDuringFlush() { + final Windowed a = new Windowed<>(keyA, new SessionWindow(2, 4)); + final Windowed b = new Windowed<>(keyA, new SessionWindow(1, 2)); + final Windowed aDeserialized = new Windowed<>("a", new SessionWindow(2, 4)); + final Windowed bDeserialized = new Windowed<>("a", new SessionWindow(1, 2)); + final CacheFlushListenerStub, String> flushListener = + new CacheFlushListenerStub<>( + new SessionWindowedDeserializer<>(new StringDeserializer()), + new StringDeserializer()); + cachingStore.setFlushListener(flushListener, true); + + cachingStore.put(b, "1".getBytes()); + cachingStore.flush(); + + assertEquals( + Collections.singletonList( + new KeyValueTimestamp<>( + bDeserialized, + new Change<>("1", null), + DEFAULT_TIMESTAMP)), + flushListener.forwarded + ); + flushListener.forwarded.clear(); + + cachingStore.put(a, "1".getBytes()); + cachingStore.flush(); + + assertEquals( + Collections.singletonList( + new KeyValueTimestamp<>( + aDeserialized, + new Change<>("1", null), + DEFAULT_TIMESTAMP)), + flushListener.forwarded + ); + flushListener.forwarded.clear(); + + cachingStore.put(a, "2".getBytes()); + cachingStore.flush(); + + assertEquals( + Collections.singletonList( + new KeyValueTimestamp<>( + aDeserialized, + new Change<>("2", "1"), + DEFAULT_TIMESTAMP)), + flushListener.forwarded + ); + flushListener.forwarded.clear(); + + cachingStore.remove(a); + cachingStore.flush(); + + assertEquals( + Collections.singletonList( + new KeyValueTimestamp<>( + aDeserialized, + new Change<>(null, "2"), + DEFAULT_TIMESTAMP)), + flushListener.forwarded + ); + flushListener.forwarded.clear(); + + cachingStore.put(a, "1".getBytes()); + cachingStore.put(a, "2".getBytes()); + cachingStore.remove(a); + cachingStore.flush(); + + assertEquals( + Collections.emptyList(), + flushListener.forwarded + ); + flushListener.forwarded.clear(); + } + + @Test + public void shouldNotForwardChangedValuesDuringFlushWhenSendOldValuesDisabled() { + final Windowed a = new Windowed<>(keyA, new SessionWindow(0, 0)); + final Windowed aDeserialized = new Windowed<>("a", new SessionWindow(0, 0)); + final CacheFlushListenerStub, String> flushListener = + new CacheFlushListenerStub<>( + new SessionWindowedDeserializer<>(new StringDeserializer()), + new StringDeserializer()); + cachingStore.setFlushListener(flushListener, false); + + cachingStore.put(a, "1".getBytes()); + cachingStore.flush(); + + cachingStore.put(a, "2".getBytes()); + cachingStore.flush(); + + cachingStore.remove(a); + cachingStore.flush(); + + assertEquals( + asList(new KeyValueTimestamp<>( + aDeserialized, + new Change<>("1", null), + DEFAULT_TIMESTAMP), + new KeyValueTimestamp<>( + aDeserialized, + new Change<>("2", null), + DEFAULT_TIMESTAMP), + new KeyValueTimestamp<>( + aDeserialized, + new Change<>(null, null), + DEFAULT_TIMESTAMP)), + flushListener.forwarded + ); + flushListener.forwarded.clear(); + + cachingStore.put(a, "1".getBytes()); + cachingStore.put(a, "2".getBytes()); + cachingStore.remove(a); + cachingStore.flush(); + + assertEquals( + Collections.emptyList(), + flushListener.forwarded + ); + flushListener.forwarded.clear(); + } + + @Test + public void shouldReturnSameResultsForSingleKeyFindSessionsAndEqualKeyRangeFindSessions() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 1)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(2, 3)), "2".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(4, 5)), "3".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(6, 7)), "4".getBytes()); + + final KeyValueIterator, byte[]> singleKeyIterator = cachingStore.findSessions(keyAA, 0L, 10L); + final KeyValueIterator, byte[]> keyRangeIterator = cachingStore.findSessions(keyAA, keyAA, 0L, 10L); + + assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); + assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); + assertFalse(singleKeyIterator.hasNext()); + assertFalse(keyRangeIterator.hasNext()); + } + + @Test + public void shouldReturnSameResultsForSingleKeyFindSessionsBackwardsAndEqualKeyRangeFindSessions() { + cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 1)), "1".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(2, 3)), "2".getBytes()); + cachingStore.put(new Windowed<>(keyAA, new SessionWindow(4, 5)), "3".getBytes()); + cachingStore.put(new Windowed<>(keyB, new SessionWindow(6, 7)), "4".getBytes()); + + final KeyValueIterator, byte[]> singleKeyIterator = + cachingStore.backwardFindSessions(keyAA, 0L, 10L); + final KeyValueIterator, byte[]> keyRangeIterator = + cachingStore.backwardFindSessions(keyAA, keyAA, 0L, 10L); + + assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); + assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); + assertFalse(singleKeyIterator.hasNext()); + assertFalse(keyRangeIterator.hasNext()); + } + + @Test + public void shouldClearNamespaceCacheOnClose() { + final Windowed a1 = new Windowed<>(keyA, new SessionWindow(0, 0)); + cachingStore.put(a1, "1".getBytes()); + assertEquals(1, cache.size()); + cachingStore.close(); + assertEquals(0, cache.size()); + } + + @Test + public void shouldThrowIfTryingToFetchFromClosedCachingStore() { + cachingStore.close(); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.fetch(keyA)); + } + + @Test + public void shouldThrowIfTryingToFindMergeSessionFromClosedCachingStore() { + cachingStore.close(); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.findSessions(keyA, 0, Long.MAX_VALUE)); + } + + @Test + public void shouldThrowIfTryingToRemoveFromClosedCachingStore() { + cachingStore.close(); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.remove(new Windowed<>(keyA, new SessionWindow(0, 0)))); + } + + @Test + public void shouldThrowIfTryingToPutIntoClosedCachingStore() { + cachingStore.close(); + assertThrows(InvalidStateStoreException.class, () -> cachingStore.put(new Windowed<>(keyA, new SessionWindow(0, 0)), "1".getBytes())); + } + + @Test + public void shouldThrowNullPointerExceptionOnFindSessionsNullKey() { + assertThrows(NullPointerException.class, () -> cachingStore.findSessions(null, 1L, 2L)); + } + + @Test + public void shouldThrowNullPointerExceptionOnFindSessionsNullFromKey() { + assertThrows(NullPointerException.class, () -> cachingStore.findSessions(null, keyA, 1L, 2L)); + } + + @Test + public void shouldThrowNullPointerExceptionOnFindSessionsNullToKey() { + assertThrows(NullPointerException.class, () -> cachingStore.findSessions(keyA, null, 1L, 2L)); + } + + @Test + public void shouldThrowNullPointerExceptionOnFetchNullFromKey() { + assertThrows(NullPointerException.class, () -> cachingStore.fetch(null, keyA)); + } + + @Test + public void shouldThrowNullPointerExceptionOnFetchNullToKey() { + assertThrows(NullPointerException.class, () -> cachingStore.fetch(keyA, null)); + } + + @Test + public void shouldThrowNullPointerExceptionOnFetchNullKey() { + assertThrows(NullPointerException.class, () -> cachingStore.fetch(null)); + } + + @Test + public void shouldThrowNullPointerExceptionOnRemoveNullKey() { + assertThrows(NullPointerException.class, () -> cachingStore.remove(null)); + } + + @Test + public void shouldThrowNullPointerExceptionOnPutNullKey() { + assertThrows(NullPointerException.class, () -> cachingStore.put(null, "1".getBytes())); + } + + @Test + public void shouldNotThrowInvalidRangeExceptionWhenBackwardWithNegativeFromKey() { + final Bytes keyFrom = Bytes.wrap(Serdes.Integer().serializer().serialize("", -1)); + final Bytes keyTo = Bytes.wrap(Serdes.Integer().serializer().serialize("", 1)); + + try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(CachingSessionStore.class)) { + final KeyValueIterator, byte[]> iterator = cachingStore.backwardFindSessions(keyFrom, keyTo, 0L, 10L); + assertFalse(iterator.hasNext()); + + final List messages = appender.getMessages(); + assertThat( + messages, + hasItem("Returning empty iterator for fetch with invalid key range: from > to." + + " This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + + " Note that the built-in numerical serdes do not follow this for negative numbers") + ); + } + } + + @Test + public void shouldNotThrowInvalidRangeExceptionWithNegativeFromKey() { + final Bytes keyFrom = Bytes.wrap(Serdes.Integer().serializer().serialize("", -1)); + final Bytes keyTo = Bytes.wrap(Serdes.Integer().serializer().serialize("", 1)); + + try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(CachingSessionStore.class)) { + final KeyValueIterator, byte[]> iterator = cachingStore.findSessions(keyFrom, keyTo, 0L, 10L); + assertFalse(iterator.hasNext()); + + final List messages = appender.getMessages(); + assertThat( + messages, + hasItem("Returning empty iterator for fetch with invalid key range: from > to." + + " This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + + " Note that the built-in numerical serdes do not follow this for negative numbers") + ); + } + } + + private List, byte[]>> addSessionsUntilOverflow(final String... sessionIds) { + final Random random = new Random(); + final List, byte[]>> results = new ArrayList<>(); + while (cache.size() == results.size()) { + final String sessionId = sessionIds[random.nextInt(sessionIds.length)]; + addSingleSession(sessionId, results); + } + return results; + } + + private void addSingleSession(final String sessionId, final List, byte[]>> allSessions) { + final int timestamp = allSessions.size() * 10; + final Windowed key = new Windowed<>(Bytes.wrap(sessionId.getBytes()), new SessionWindow(timestamp, timestamp)); + final byte[] value = "1".getBytes(); + cachingStore.put(key, value); + allSessions.add(KeyValue.pair(key, value)); + } + + public static class CacheFlushListenerStub implements CacheFlushListener { + final Deserializer keyDeserializer; + final Deserializer valueDesializer; + final List>> forwarded = new LinkedList<>(); + + CacheFlushListenerStub(final Deserializer keyDeserializer, + final Deserializer valueDesializer) { + this.keyDeserializer = keyDeserializer; + this.valueDesializer = valueDesializer; + } + + @Override + public void apply(final byte[] key, + final byte[] newValue, + final byte[] oldValue, + final long timestamp) { + forwarded.add( + new KeyValueTimestamp<>( + keyDeserializer.deserialize(null, key), + new Change<>( + valueDesializer.deserialize(null, newValue), + valueDesializer.deserialize(null, oldValue)), + timestamp)); + } + } +} diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentWindowStoreTest.java similarity index 99% rename from streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java rename to streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentWindowStoreTest.java index 42b750b932c37..507b553b5327e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentWindowStoreTest.java @@ -75,7 +75,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -public class CachingWindowStoreTest { +public class CachingPersistentWindowStoreTest { private static final int MAX_CACHE_SIZE_BYTES = 150; private static final long DEFAULT_TIMESTAMP = 10L; @@ -88,7 +88,7 @@ public class CachingWindowStoreTest { private RocksDBSegmentedBytesStore bytesStore; private WindowStore underlyingStore; private CachingWindowStore cachingStore; - private CachingKeyValueStoreTest.CacheFlushListenerStub, String> cacheListener; + private CacheFlushListenerStub, String> cacheListener; private ThreadCache cache; private WindowKeySchema keySchema; @@ -99,7 +99,7 @@ public void setUp() { underlyingStore = new RocksDBWindowStore(bytesStore, false, WINDOW_SIZE); final TimeWindowedDeserializer keyDeserializer = new TimeWindowedDeserializer<>(new StringDeserializer(), WINDOW_SIZE); keyDeserializer.setIsChangelogTopic(true); - cacheListener = new CachingKeyValueStoreTest.CacheFlushListenerStub<>(keyDeserializer, new StringDeserializer()); + cacheListener = new CacheFlushListenerStub<>(keyDeserializer, new StringDeserializer()); cachingStore = new CachingWindowStore(underlyingStore, WINDOW_SIZE, SEGMENT_INTERVAL); cachingStore.setFlushListener(cacheListener, false); cache = new ThreadCache(new LogContext("testCache "), MAX_CACHE_SIZE_BYTES, new MockStreamsMetrics(new Metrics())); From b8562422783d83ad5d334155d1bcc6c56cea47d2 Mon Sep 17 00:00:00 2001 From: John Roesler Date: Wed, 7 Oct 2020 22:35:16 -0500 Subject: [PATCH 12/14] fix formatting --- .../kafka/connect/runtime/SessionKey.java | 3 +- .../streams/state/ReadOnlySessionStore.java | 31 ++--- .../kafka/streams/state/SessionStore.java | 2 - .../state/internals/CachingSessionStore.java | 59 ++++++---- .../ChangeLoggingSessionBytesStore.java | 8 +- .../CompositeReadOnlySessionStore.java | 89 ++++++++------ .../state/internals/InMemorySessionStore.java | 96 +++++++-------- ...MergedSortedCacheSessionStoreIterator.java | 4 +- .../state/internals/MeteredSessionStore.java | 18 ++- .../state/internals/RocksDBSessionStore.java | 1 + .../state/internals/SessionKeySchema.java | 1 + .../AbstractSessionBytesStoreTest.java | 34 +++--- .../internals/CacheFlushListenerStub.java | 8 +- .../CachingInMemorySessionStoreTest.java | 15 +-- .../CachingPersistentSessionStoreTest.java | 110 +++++++++++------- .../ChangeLoggingSessionBytesStoreTest.java | 2 + ...dCacheWrappedSessionStoreIteratorTest.java | 2 +- .../internals/MeteredSessionStoreTest.java | 44 ++++--- .../kafka/test/ReadOnlySessionStoreStub.java | 11 +- 19 files changed, 313 insertions(+), 225 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java index 2ff0cc84c5ac1..ab5476e4b9000 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java @@ -29,8 +29,7 @@ public class SessionKey { /** * Create a new session key with the given key value and creation timestamp - * - * @param key the actual cryptographic key to use for request validation; may not be null + * @param key the actual cryptographic key to use for request validation; may not be null * @param creationTimestamp the time at which the key was generated */ public SessionKey(SecretKey key, long creationTimestamp) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java index 5f7654ca201ef..8874908d18072 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java @@ -24,7 +24,7 @@ * Implementations should be thread-safe as concurrent reads and writes * are expected. * - * @param the key type + * @param the key type * @param the aggregated value type */ public interface ReadOnlySessionStore { @@ -44,7 +44,7 @@ public interface ReadOnlySessionStore { default KeyValueIterator, AGG> findSessions(final K key, final long earliestSessionEndTime, final long latestSessionStartTime) { - throw new UnsupportedOperationException("Moved from SessionStore"); + throw new UnsupportedOperationException("This API is not supported by this implementation of ReadOnlySessionStore."); } /** @@ -62,7 +62,7 @@ default KeyValueIterator, AGG> findSessions(final K key, default KeyValueIterator, AGG> backwardFindSessions(final K key, final long earliestSessionEndTime, final long latestSessionStartTime) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("This API is not supported by this implementation of ReadOnlySessionStore."); } /** @@ -82,7 +82,7 @@ default KeyValueIterator, AGG> findSessions(final K keyFrom, final K keyTo, final long earliestSessionEndTime, final long latestSessionStartTime) { - throw new UnsupportedOperationException("Moved from SessionStore"); + throw new UnsupportedOperationException("This API is not supported by this implementation of ReadOnlySessionStore."); } @@ -103,7 +103,7 @@ default KeyValueIterator, AGG> backwardFindSessions(final K keyFrom, final K keyTo, final long earliestSessionEndTime, final long latestSessionStartTime) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("This API is not supported by this implementation of ReadOnlySessionStore."); } /** @@ -116,7 +116,7 @@ default KeyValueIterator, AGG> backwardFindSessions(final K keyFrom, * @throws NullPointerException If {@code null} is used for any key. */ default AGG fetchSession(final K key, final long startTime, final long endTime) { - throw new UnsupportedOperationException("Moved from SessionStore"); + throw new UnsupportedOperationException("This API is not supported by this implementation of ReadOnlySessionStore."); } /** @@ -126,9 +126,10 @@ default AGG fetchSession(final K key, final long startTime, final long endTime) * For each key, the iterator guarantees ordering of sessions, starting from the oldest/earliest * available session to the newest/latest session. * - * @param key record key to find aggregated session values for - * @return KeyValueIterator containing all sessions for the provided key, from oldest to newest session. - * @throws NullPointerException If null is used for key. + * @param key record key to find aggregated session values for + * @return KeyValueIterator containing all sessions for the provided key, from oldest to newest session. + * @throws NullPointerException If null is used for key. + * */ KeyValueIterator, AGG> fetch(final K key); @@ -144,7 +145,7 @@ default AGG fetchSession(final K key, final long startTime, final long endTime) * @throws NullPointerException If null is used for key. */ default KeyValueIterator, AGG> backwardFetch(final K key) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("This API is not supported by this implementation of ReadOnlySessionStore."); } /** @@ -154,10 +155,10 @@ default KeyValueIterator, AGG> backwardFetch(final K key) { * For each key, the iterator guarantees ordering of sessions, starting from the oldest/earliest * available session to the newest/latest session. * - * @param from first key in the range to find aggregated session values for - * @param to last key in the range to find aggregated session values for - * @return KeyValueIterator containing all sessions for the provided key, from oldest to newest session. - * @throws NullPointerException If null is used for any of the keys. + * @param from first key in the range to find aggregated session values for + * @param to last key in the range to find aggregated session values for + * @return KeyValueIterator containing all sessions for the provided key, from oldest to newest session. + * @throws NullPointerException If null is used for any of the keys. */ KeyValueIterator, AGG> fetch(final K from, final K to); @@ -174,6 +175,6 @@ default KeyValueIterator, AGG> backwardFetch(final K key) { * @throws NullPointerException If null is used for any of the keys. */ default KeyValueIterator, AGG> backwardFetch(final K from, final K to) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("This API is not supported by this implementation of ReadOnlySessionStore."); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java index 35e14e6407fe7..47f48d5e8143c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java @@ -36,7 +36,6 @@ public interface SessionStore extends StateStore, ReadOnlySessionStore extends StateStore, ReadOnlySessionStore, byte[]> findSessions(final Bytes key, final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, true) : context.cache().range(cacheName, - cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), - cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) + cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), + cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) ); - final KeyValueIterator, byte[]> storeIterator = - wrapped().findSessions(key, earliestSessionEndTime, latestSessionStartTime); - final HasNextCondition hasNextCondition = - keySchema.hasNextCondition(key, key, earliestSessionEndTime, latestSessionStartTime); + final KeyValueIterator, byte[]> storeIterator = wrapped().findSessions(key, + earliestSessionEndTime, + latestSessionStartTime); + final HasNextCondition hasNextCondition = keySchema.hasNextCondition(key, + key, + earliestSessionEndTime, + latestSessionStartTime); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, true); @@ -172,18 +175,24 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte final PeekingKeyValueIterator cacheIterator = wrapped().persistent() ? new CacheIteratorWrapper(key, earliestSessionEndTime, latestSessionStartTime, false) : - context.cache().reverseRange(cacheName, + context.cache().reverseRange( + cacheName, cacheFunction.cacheKey(keySchema.lowerRangeFixedSize(key, earliestSessionEndTime)), - cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime)) + cacheFunction.cacheKey(keySchema.upperRangeFixedSize(key, latestSessionStartTime) + ) ); - final KeyValueIterator, byte[]> storeIterator = wrapped().backwardFindSessions(key, + final KeyValueIterator, byte[]> storeIterator = wrapped().backwardFindSessions( + key, earliestSessionEndTime, - latestSessionStartTime); - final HasNextCondition hasNextCondition = keySchema.hasNextCondition(key, + latestSessionStartTime + ); + final HasNextCondition hasNextCondition = keySchema.hasNextCondition( + key, key, earliestSessionEndTime, - latestSessionStartTime); + latestSessionStartTime + ); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, false); @@ -211,8 +220,10 @@ public KeyValueIterator, byte[]> findSessions(final Bytes keyFro final KeyValueIterator, byte[]> storeIterator = wrapped().findSessions( keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime ); - final HasNextCondition hasNextCondition = - keySchema.hasNextCondition(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + final HasNextCondition hasNextCondition = keySchema.hasNextCondition(keyFrom, + keyTo, + earliestSessionEndTime, + latestSessionStartTime); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, true); @@ -225,9 +236,9 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte final long latestSessionStartTime) { if (keyFrom.compareTo(keyTo) > 0) { LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. " + - "This may be due to range arguments set in the wrong order, " + - "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " + - "Note that the built-in numerical serdes do not follow this for negative numbers"); + "This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " + + "Note that the built-in numerical serdes do not follow this for negative numbers"); return KeyValueIterators.emptyIterator(); } @@ -237,13 +248,14 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte final Bytes cacheKeyTo = cacheFunction.cacheKey(keySchema.upperRange(keyTo, latestSessionStartTime)); final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = context.cache().reverseRange(cacheName, cacheKeyFrom, cacheKeyTo); - final KeyValueIterator, byte[]> storeIterator = wrapped().backwardFindSessions( - keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime - ); - final HasNextCondition hasNextCondition = keySchema.hasNextCondition(keyFrom, + final KeyValueIterator, byte[]> storeIterator = + wrapped().backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + final HasNextCondition hasNextCondition = keySchema.hasNextCondition( + keyFrom, keyTo, earliestSessionEndTime, - latestSessionStartTime); + latestSessionStartTime + ); final PeekingKeyValueIterator filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition, cacheFunction); return new MergedSortedCacheSessionStoreIterator(filteredCacheIterator, storeIterator, cacheFunction, false); @@ -313,7 +325,7 @@ public void close() { ); if (!suppressed.isEmpty()) { throwSuppressed("Caught an exception while closing caching session store for store " + name(), - suppressed); + suppressed); } } @@ -327,6 +339,7 @@ private class CacheIteratorWrapper implements PeekingKeyValueIterator, byte[]> findSessions(final Bytes key, f } @Override - public KeyValueIterator, byte[]> backwardFindSessions(final Bytes key, final long earliestSessionEndTime, final long latestSessionStartTime) { + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes key, + final long earliestSessionEndTime, + final long latestSessionStartTime) { return wrapped().backwardFindSessions(key, earliestSessionEndTime, latestSessionStartTime); } @@ -60,7 +62,9 @@ public KeyValueIterator, byte[]> findSessions(final Bytes keyFro } @Override - public KeyValueIterator, byte[]> backwardFindSessions(final Bytes keyFrom, final Bytes keyTo, final long earliestSessionEndTime, final long latestSessionStartTime) { + public KeyValueIterator, byte[]> backwardFindSessions(final Bytes keyFrom, final Bytes keyTo, + final long earliestSessionEndTime, + final long latestSessionStartTime) { return wrapped().backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java index 46f3e115885de..72233122a6286 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java @@ -50,17 +50,21 @@ public KeyValueIterator, V> findSessions(final K key, final List> stores = storeProvider.stores(storeName, queryableStoreType); for (final ReadOnlySessionStore store : stores) { try { - final KeyValueIterator, V> result = store.findSessions(key, earliestSessionEndTime, latestSessionStartTime); + final KeyValueIterator, V> result = + store.findSessions(key, earliestSessionEndTime, latestSessionStartTime); + if (!result.hasNext()) { result.close(); } else { return result; } } catch (final InvalidStateStoreException ise) { - throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + throw new InvalidStateStoreException( + "State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata.", + ise + ); } } return KeyValueIterators.emptyIterator(); @@ -81,10 +85,12 @@ public KeyValueIterator, V> backwardFindSessions(final K key, return result; } } catch (final InvalidStateStoreException ise) { - throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + throw new InvalidStateStoreException( + "State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata.", + ise + ); } } return KeyValueIterators.emptyIterator(); @@ -100,17 +106,20 @@ public KeyValueIterator, V> findSessions(final K keyFrom, final List> stores = storeProvider.stores(storeName, queryableStoreType); for (final ReadOnlySessionStore store : stores) { try { - final KeyValueIterator, V> result = store.findSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + final KeyValueIterator, V> result = + store.findSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); if (!result.hasNext()) { result.close(); } else { return result; } } catch (final InvalidStateStoreException ise) { - throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + throw new InvalidStateStoreException( + "State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata.", + ise + ); } } return KeyValueIterators.emptyIterator(); @@ -126,17 +135,20 @@ public KeyValueIterator, V> backwardFindSessions(final K keyFrom, final List> stores = storeProvider.stores(storeName, queryableStoreType); for (final ReadOnlySessionStore store : stores) { try { - final KeyValueIterator, V> result = store.backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); + final KeyValueIterator, V> result = + store.backwardFindSessions(keyFrom, keyTo, earliestSessionEndTime, latestSessionStartTime); if (!result.hasNext()) { result.close(); } else { return result; } } catch (final InvalidStateStoreException ise) { - throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + throw new InvalidStateStoreException( + "State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata.", + ise + ); } } return KeyValueIterators.emptyIterator(); @@ -150,10 +162,12 @@ public V fetchSession(final K key, final long startTime, final long endTime) { try { return store.fetchSession(key, startTime, endTime); } catch (final InvalidStateStoreException ise) { - throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + throw new InvalidStateStoreException( + "State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata.", + ise + ); } } return null; @@ -173,9 +187,9 @@ public KeyValueIterator, V> fetch(final K key) { } } catch (final InvalidStateStoreException ise) { throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata. " + + "Original error message: " + ise.toString()); } } return KeyValueIterators.emptyIterator(); @@ -194,10 +208,12 @@ public KeyValueIterator, V> backwardFetch(final K key) { return result; } } catch (final InvalidStateStoreException ise) { - throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" + - " and may have been migrated to another instance; " + - "please re-discover its location from the state metadata. " + - "Original error message: " + ise.toString()); + throw new InvalidStateStoreException( + "State store [" + storeName + "] is not available anymore" + + " and may have been migrated to another instance; " + + "please re-discover its location from the state metadata.", + ise + ); } } return KeyValueIterators.emptyIterator(); @@ -209,9 +225,9 @@ public KeyValueIterator, V> fetch(final K from, final K to) { Objects.requireNonNull(to, "to can't be null"); final NextIteratorFunction, V, ReadOnlySessionStore> nextIteratorFunction = store -> store.fetch(from, to); return new DelegatingPeekingKeyValueIterator<>(storeName, - new CompositeKeyValueIterator<>( - storeProvider.stores(storeName, queryableStoreType).iterator(), - nextIteratorFunction)); + new CompositeKeyValueIterator<>( + storeProvider.stores(storeName, queryableStoreType).iterator(), + nextIteratorFunction)); } @Override @@ -219,9 +235,12 @@ public KeyValueIterator, V> backwardFetch(final K from, final K to) Objects.requireNonNull(from, "from can't be null"); Objects.requireNonNull(to, "to can't be null"); final NextIteratorFunction, V, ReadOnlySessionStore> nextIteratorFunction = store -> store.backwardFetch(from, to); - return new DelegatingPeekingKeyValueIterator<>(storeName, + return new DelegatingPeekingKeyValueIterator<>( + storeName, new CompositeKeyValueIterator<>( storeProvider.stores(storeName, queryableStoreType).iterator(), - nextIteratorFunction)); + nextIteratorFunction + ) + ); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java index 3aad2d1e0458d..913432eabf994 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java @@ -55,7 +55,7 @@ public class InMemorySessionStore implements SessionStore { private final long retentionPeriod; private final ConcurrentNavigableMap>> endTimeMap = new ConcurrentSkipListMap<>(); - private final Set openIterators = ConcurrentHashMap.newKeySet(); + private final Set openIterators = ConcurrentHashMap.newKeySet(); private volatile boolean open = false; @@ -164,12 +164,11 @@ public KeyValueIterator, byte[]> findSessions(final Bytes key, removeExpiredSegments(); - return registerNewIterator( - key, - key, - latestSessionStartTime, - endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator(), - true); + return registerNewIterator(key, + key, + latestSessionStartTime, + endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator(), + true); } @Override @@ -185,7 +184,8 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte key, latestSessionStartTime, endTimeMap.tailMap(earliestSessionEndTime, true).descendingMap().entrySet().iterator(), - false); + false + ); } @Override @@ -206,12 +206,11 @@ public KeyValueIterator, byte[]> findSessions(final Bytes keyFro return KeyValueIterators.emptyIterator(); } - return registerNewIterator( - keyFrom, - keyTo, - latestSessionStartTime, - endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator(), - true); + return registerNewIterator(keyFrom, + keyTo, + latestSessionStartTime, + endTimeMap.tailMap(earliestSessionEndTime, true).entrySet().iterator(), + true); } @Override @@ -237,7 +236,8 @@ public KeyValueIterator, byte[]> backwardFindSessions(final Byte keyTo, latestSessionStartTime, endTimeMap.tailMap(earliestSessionEndTime, true).descendingMap().entrySet().iterator(), - false); + false + ); } @Override @@ -247,12 +247,7 @@ public KeyValueIterator, byte[]> fetch(final Bytes key) { removeExpiredSegments(); - return registerNewIterator( - key, - key, - Long.MAX_VALUE, - endTimeMap.entrySet().iterator(), - true); + return registerNewIterator(key, key, Long.MAX_VALUE, endTimeMap.entrySet().iterator(), true); } @Override @@ -262,12 +257,7 @@ public KeyValueIterator, byte[]> backwardFetch(final Bytes key) removeExpiredSegments(); - return registerNewIterator( - key, - key, - Long.MAX_VALUE, - endTimeMap.descendingMap().entrySet().iterator(), - false); + return registerNewIterator(key, key, Long.MAX_VALUE, endTimeMap.descendingMap().entrySet().iterator(), false); } @Override @@ -278,6 +268,7 @@ public KeyValueIterator, byte[]> fetch(final Bytes from, final B removeExpiredSegments(); + return registerNewIterator(from, to, Long.MAX_VALUE, endTimeMap.entrySet().iterator(), false); } @@ -342,7 +333,8 @@ private InMemorySessionStoreIterator registerNewIterator(final Bytes keyFrom, latestSessionStartTime, endTimeIterator, openIterators::remove, - forward); + forward + ); openIterators.add(iterator); return iterator; } @@ -353,6 +345,7 @@ interface ClosingCallback { private static class InMemorySessionStoreIterator implements KeyValueIterator, byte[]> { + private final Iterator>>> endTimeIterator; private Iterator>> keyIterator; private Iterator> recordIterator; @@ -363,7 +356,6 @@ private static class InMemorySessionStoreIterator implements KeyValueIterator>>> endTimeIterator; private final ClosingCallback callback; @@ -378,6 +370,7 @@ private static class InMemorySessionStoreIterator implements KeyValueIterator>> nextEndTimeEntry = endTimeIterator.next(); currentEndTime = nextEndTimeEntry.getKey(); - final Set>> entries; if (forward) { - entries = nextEndTimeEntry.getValue() - .subMap(keyFrom, true, keyTo, true) - .entrySet(); + keyIterator = nextEndTimeEntry.getValue() + .subMap(keyFrom, true, keyTo, true) + .entrySet() + .iterator(); } else { - entries = nextEndTimeEntry.getValue() - .subMap(keyFrom, true, keyTo, true) - .descendingMap() - .entrySet(); + keyIterator = nextEndTimeEntry.getValue() + .subMap(keyFrom, true, keyTo, true) + .descendingMap() + .entrySet() + .iterator(); } - keyIterator = entries.iterator(); if (setInnerIterators()) { return; @@ -479,23 +472,22 @@ private boolean setInnerIterators() { currentKey = nextKeyEntry.getKey(); if (latestSessionStartTime == Long.MAX_VALUE) { - final Set> entries; - if (forward) entries = nextKeyEntry.getValue().descendingMap().entrySet(); - else entries = nextKeyEntry.getValue().entrySet(); - recordIterator = entries.iterator(); + if (forward) { + recordIterator = nextKeyEntry.getValue().descendingMap().entrySet().iterator(); + } else { + recordIterator = nextKeyEntry.getValue().entrySet().iterator(); + } } else { - final Set> entries; if (forward) { - entries = nextKeyEntry.getValue() - .headMap(latestSessionStartTime, true) - .descendingMap() - .entrySet(); + recordIterator = nextKeyEntry.getValue() + .headMap(latestSessionStartTime, true) + .descendingMap() + .entrySet().iterator(); } else { - entries = nextKeyEntry.getValue() - .headMap(latestSessionStartTime, true) - .entrySet(); + recordIterator = nextKeyEntry.getValue() + .headMap(latestSessionStartTime, true) + .entrySet().iterator(); } - recordIterator = entries.iterator(); } if (recordIterator.hasNext()) { @@ -511,7 +503,9 @@ private void getNextIterators() { if (setInnerIterators()) { return; } + setAllIterators(); } } + } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java index 5a9204071d697..cd0c0df2c2f57 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheSessionStoreIterator.java @@ -24,9 +24,9 @@ /** * Merges two iterators. Assumes each of them is sorted by key + * */ -class MergedSortedCacheSessionStoreIterator - extends AbstractMergedSortedCacheStoreIterator, Windowed, byte[], byte[]> { +class MergedSortedCacheSessionStoreIterator extends AbstractMergedSortedCacheStoreIterator, Windowed, byte[], byte[]> { private final SegmentedCacheFunction cacheFunction; diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java index 12150838c53d0..ef8d91ff20325 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java @@ -193,7 +193,8 @@ public KeyValueIterator, V> backwardFetch(final K key) { fetchSensor, streamsMetrics, serdes, - time); + time + ); } @Override @@ -219,7 +220,8 @@ public KeyValueIterator, V> backwardFetch(final K from, fetchSensor, streamsMetrics, serdes, - time); + time + ); } @Override @@ -249,11 +251,13 @@ public KeyValueIterator, V> backwardFindSessions(final K key, wrapped().backwardFindSessions( bytesKey, earliestSessionEndTime, - latestSessionStartTime), + latestSessionStartTime + ), fetchSensor, streamsMetrics, serdes, - time); + time + ); } @Override @@ -291,11 +295,13 @@ public KeyValueIterator, V> backwardFindSessions(final K keyFrom, bytesKeyFrom, bytesKeyTo, earliestSessionEndTime, - latestSessionStartTime), + latestSessionStartTime + ), fetchSensor, streamsMetrics, serdes, - time); + time + ); } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java index e208ac12d56d3..338769abea4a5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java @@ -21,6 +21,7 @@ import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.SessionStore; + public class RocksDBSessionStore extends WrappedStateStore implements SessionStore { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java index f3c452495839b..326b86945f8e4 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.util.List; + public class SessionKeySchema implements SegmentedBytesStore.KeySchema { private static final int TIMESTAMP_SIZE = 8; diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java index 8130e02e66523..b355f0e34dacd 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractSessionBytesStoreTest.java @@ -66,6 +66,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; + public abstract class AbstractSessionBytesStoreTest { static final long SEGMENT_INTERVAL = 60_000L; @@ -119,16 +120,16 @@ public void shouldPutAndFindSessionsInRange() { final List, Long>> expected = Arrays.asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L)); - try (final KeyValueIterator, Long> values = - sessionStore.findSessions(key, 0, 1000L)) { + try (final KeyValueIterator, Long> values = sessionStore.findSessions(key, 0, 1000L) + ) { assertEquals(new HashSet<>(expected), toSet(values)); } final List, Long>> expected2 = Collections.singletonList(KeyValue.pair(a2, 2L)); - try (final KeyValueIterator, Long> values2 = - sessionStore.findSessions(key, 400L, 600L)) { + try (final KeyValueIterator, Long> values2 = sessionStore.findSessions(key, 400L, 600L) + ) { assertEquals(new HashSet<>(expected2), toSet(values2)); } } @@ -144,18 +145,16 @@ public void shouldPutAndBackwardFindSessionsInRange() { sessionStore.put(new Windowed<>(key, new SessionWindow(2500L, 3000L)), 2L); final List, Long>> expected = - Arrays.asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L)); + asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L)); - try (final KeyValueIterator, Long> values = - sessionStore.backwardFindSessions(key, 0, 1000L)) { + try (final KeyValueIterator, Long> values = sessionStore.backwardFindSessions(key, 0, 1000L)) { assertEquals(new HashSet<>(expected), toSet(values)); } final List, Long>> expected2 = Collections.singletonList(KeyValue.pair(a2, 2L)); - try (final KeyValueIterator, Long> values2 = - sessionStore.backwardFindSessions(key, 400L, 600L)) { + try (final KeyValueIterator, Long> values2 = sessionStore.backwardFindSessions(key, 400L, 600L)) { assertEquals(new HashSet<>(expected2), toSet(values2)); } } @@ -182,11 +181,12 @@ public void shouldFetchAllSessionsWithSameRecordKey() { @Test public void shouldBackwardFetchAllSessionsWithSameRecordKey() { - final List, Long>> expected = Arrays.asList( + final List, Long>> expected = asList( KeyValue.pair(new Windowed<>("a", new SessionWindow(0, 0)), 1L), KeyValue.pair(new Windowed<>("a", new SessionWindow(10, 10)), 2L), KeyValue.pair(new Windowed<>("a", new SessionWindow(100, 100)), 3L), - KeyValue.pair(new Windowed<>("a", new SessionWindow(1000, 1000)), 4L)); + KeyValue.pair(new Windowed<>("a", new SessionWindow(1000, 1000)), 4L) + ); for (final KeyValue, Long> kv : expected) { sessionStore.put(kv.key, kv.value); @@ -224,12 +224,13 @@ public void shouldFetchAllSessionsWithinKeyRange() { @Test public void shouldBackwardFetchAllSessionsWithinKeyRange() { - final List, Long>> expected = Arrays.asList( + final List, Long>> expected = asList( KeyValue.pair(new Windowed<>("aa", new SessionWindow(10, 10)), 2L), KeyValue.pair(new Windowed<>("b", new SessionWindow(1000, 1000)), 4L), KeyValue.pair(new Windowed<>("aaa", new SessionWindow(100, 100)), 3L), - KeyValue.pair(new Windowed<>("bb", new SessionWindow(1500, 2000)), 5L)); + KeyValue.pair(new Windowed<>("bb", new SessionWindow(1500, 2000)), 5L) + ); for (final KeyValue, Long> kv : expected) { sessionStore.put(kv.key, kv.value); @@ -282,9 +283,10 @@ public void shouldBackwardFindValuesWithinMergingSessionWindowRange() { sessionStore.put(new Windowed<>(key, new SessionWindow(0L, 0L)), 1L); sessionStore.put(new Windowed<>(key, new SessionWindow(1000L, 1000L)), 2L); - final List, Long>> expected = Arrays.asList( + final List, Long>> expected = asList( KeyValue.pair(new Windowed<>(key, new SessionWindow(0L, 0L)), 1L), - KeyValue.pair(new Windowed<>(key, new SessionWindow(1000L, 1000L)), 2L)); + KeyValue.pair(new Windowed<>(key, new SessionWindow(1000L, 1000L)), 2L) + ); try (final KeyValueIterator, Long> results = sessionStore.backwardFindSessions(key, -1, 1000L)) { assertEquals(new HashSet<>(expected), toSet(results)); @@ -358,7 +360,7 @@ public void shouldBackwardFindSessionsToMerge() { sessionStore.put(session5, 5L); final List, Long>> expected = - Arrays.asList(KeyValue.pair(session2, 2L), KeyValue.pair(session3, 3L)); + asList(KeyValue.pair(session2, 2L), KeyValue.pair(session3, 3L)); try (final KeyValueIterator, Long> results = sessionStore.backwardFindSessions("a", 150, 300)) { assertEquals(new HashSet<>(expected), toSet(results)); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java index 11b6300f743cb..ea4b147f00fa3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CacheFlushListenerStub.java @@ -23,8 +23,8 @@ import java.util.Map; public class CacheFlushListenerStub implements CacheFlushListener { - final Deserializer keyDeserializer; - final Deserializer valueDeserializer; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; final Map> forwarded = new HashMap<>(); CacheFlushListenerStub(final Deserializer keyDeserializer, @@ -42,6 +42,8 @@ public void apply(final byte[] key, keyDeserializer.deserialize(null, key), new Change<>( valueDeserializer.deserialize(null, newValue), - valueDeserializer.deserialize(null, oldValue))); + valueDeserializer.deserialize(null, oldValue) + ) + ); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java index eadbb8b43c33c..fdcc01ab3b9ac 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java @@ -44,7 +44,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -379,7 +378,7 @@ public void shouldFetchRangeCorrectlyAcrossSegments() { keys.add(rangeResults.next().key); } rangeResults.close(); - assertEquals(Arrays.asList(a1, aa1, a2, a3, aa3), keys); + assertEquals(asList(a1, aa1, a2, a3, aa3), keys); } @Test @@ -402,7 +401,7 @@ public void shouldBackwardFetchRangeCorrectlyAcrossSegments() { keys.add(rangeResults.next().key); } rangeResults.close(); - assertEquals(Arrays.asList(aa3, a3, a2, aa1, a1), keys); + assertEquals(asList(aa3, a3, a2, aa1, a1), keys); } @Test @@ -654,10 +653,12 @@ public void shouldNotThrowInvalidRangeExceptionWhenBackwardWithNegativeFromKey() final List messages = appender.getMessages(); assertThat( messages, - hasItem("Returning empty iterator for fetch with invalid key range: from > to." + - " This may be due to range arguments set in the wrong order, " + - "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + - " Note that the built-in numerical serdes do not follow this for negative numbers") + hasItem( + "Returning empty iterator for fetch with invalid key range: from > to." + + " This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + + " Note that the built-in numerical serdes do not follow this for negative numbers" + ) ); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java index c85de42d9efd4..d472c7f5637db 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentSessionStoreTest.java @@ -44,7 +44,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -64,7 +63,6 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -@SuppressWarnings("PointlessArithmeticExpression") public class CachingPersistentSessionStoreTest { private static final int MAX_CACHE_SIZE_BYTES = 600; @@ -88,11 +86,13 @@ public void before() { "metric-scope", Long.MAX_VALUE, SEGMENT_INTERVAL, - new SessionKeySchema()); + new SessionKeySchema() + ); underlyingStore = new RocksDBSessionStore(segmented); cachingStore = new CachingSessionStore(underlyingStore, SEGMENT_INTERVAL); cache = new ThreadCache(new LogContext("testCache "), MAX_CACHE_SIZE_BYTES, new MockStreamsMetrics(new Metrics())); - final InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); + final InternalMockProcessorContext context = + new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); context.setRecordContext(new ProcessorRecordContext(DEFAULT_TIMESTAMP, 0, 0, TOPIC, null)); cachingStore.init((StateStoreContext) context, cachingStore); } @@ -110,8 +110,10 @@ public void shouldPutFetchFromCache() { assertEquals(3, cache.size()); - final KeyValueIterator, byte[]> a = cachingStore.findSessions(keyA, 0, 0); - final KeyValueIterator, byte[]> b = cachingStore.findSessions(keyB, 0, 0); + final KeyValueIterator, byte[]> a = + cachingStore.findSessions(keyA, 0, 0); + final KeyValueIterator, byte[]> b = + cachingStore.findSessions(keyB, 0, 0); verifyWindowedKeyValue(a.next(), new Windowed<>(keyA, new SessionWindow(0, 0)), "1"); verifyWindowedKeyValue(b.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); @@ -127,7 +129,8 @@ public void shouldPutFetchAllKeysFromCache() { assertEquals(3, cache.size()); - final KeyValueIterator, byte[]> all = cachingStore.findSessions(keyA, keyB, 0, 0); + final KeyValueIterator, byte[]> all = + cachingStore.findSessions(keyA, keyB, 0, 0); verifyWindowedKeyValue(all.next(), new Windowed<>(keyA, new SessionWindow(0, 0)), "1"); verifyWindowedKeyValue(all.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); verifyWindowedKeyValue(all.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); @@ -142,7 +145,8 @@ public void shouldPutBackwardFetchAllKeysFromCache() { assertEquals(3, cache.size()); - final KeyValueIterator, byte[]> all = cachingStore.backwardFindSessions(keyA, keyB, 0, 0); + final KeyValueIterator, byte[]> all = + cachingStore.backwardFindSessions(keyA, keyB, 0, 0); verifyWindowedKeyValue(all.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); verifyWindowedKeyValue(all.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); verifyWindowedKeyValue(all.next(), new Windowed<>(keyA, new SessionWindow(0, 0)), "1"); @@ -203,7 +207,8 @@ private void setUpCloseTests() { EasyMock.replay(underlyingStore); cachingStore = new CachingSessionStore(underlyingStore, SEGMENT_INTERVAL); cache = EasyMock.niceMock(ThreadCache.class); - final InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); + final InternalMockProcessorContext context = + new InternalMockProcessorContext(TestUtils.tempDirectory(), null, null, null, cache); context.setRecordContext(new ProcessorRecordContext(10, 0, 0, TOPIC, null)); cachingStore.init((StateStoreContext) context, cachingStore); } @@ -216,7 +221,8 @@ public void shouldPutFetchRangeFromCache() { assertEquals(3, cache.size()); - final KeyValueIterator, byte[]> some = cachingStore.findSessions(keyAA, keyB, 0, 0); + final KeyValueIterator, byte[]> some = + cachingStore.findSessions(keyAA, keyB, 0, 0); verifyWindowedKeyValue(some.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); verifyWindowedKeyValue(some.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); assertFalse(some.hasNext()); @@ -230,7 +236,8 @@ public void shouldPutBackwardFetchRangeFromCache() { assertEquals(3, cache.size()); - final KeyValueIterator, byte[]> some = cachingStore.backwardFindSessions(keyAA, keyB, 0, 0); + final KeyValueIterator, byte[]> some = + cachingStore.backwardFindSessions(keyAA, keyB, 0, 0); verifyWindowedKeyValue(some.next(), new Windowed<>(keyB, new SessionWindow(0, 0)), "1"); verifyWindowedKeyValue(some.next(), new Windowed<>(keyAA, new SessionWindow(0, 0)), "1"); assertFalse(some.hasNext()); @@ -279,7 +286,8 @@ public void shouldBackwardFetchAllSessionsWithSameRecordKey() { public void shouldFlushItemsToStoreOnEviction() { final List, byte[]>> added = addSessionsUntilOverflow("a", "b", "c", "d"); assertEquals(added.size() - 1, cache.size()); - final KeyValueIterator, byte[]> iterator = cachingStore.findSessions(added.get(0).key.key(), 0, 0); + final KeyValueIterator, byte[]> iterator = + cachingStore.findSessions(added.get(0).key.key(), 0, 0); final KeyValue, byte[]> next = iterator.next(); assertEquals(added.get(0).key, next.key); assertArrayEquals(added.get(0).value, next.value); @@ -291,7 +299,8 @@ public void shouldQueryItemsInCacheAndStore() { final KeyValueIterator, byte[]> iterator = cachingStore.findSessions( Bytes.wrap("a".getBytes(StandardCharsets.UTF_8)), 0, - added.size() * 10); + added.size() * 10L + ); final List, byte[]>> actual = toList(iterator); verifyKeyValueList(added, actual); } @@ -385,7 +394,7 @@ public void shouldFetchRangeCorrectlyAcrossSegments() { keys.add(rangeResults.next().key); } rangeResults.close(); - assertEquals(Arrays.asList(a1, aa1, a2, a3, aa3), keys); + assertEquals(asList(a1, aa1, a2, a3, aa3), keys); } @Test @@ -408,7 +417,7 @@ public void shouldBackwardFetchRangeCorrectlyAcrossSegments() { keys.add(rangeResults.next().key); } rangeResults.close(); - assertEquals(Arrays.asList(aa3, a3, a2, aa1, a1), keys); + assertEquals(asList(aa3, a3, a2, aa1, a1), keys); } @Test @@ -426,7 +435,8 @@ public void shouldForwardChangedValuesDuringFlush() { final CacheFlushListenerStub, String> flushListener = new CacheFlushListenerStub<>( new SessionWindowedDeserializer<>(new StringDeserializer()), - new StringDeserializer()); + new StringDeserializer() + ); cachingStore.setFlushListener(flushListener, true); cachingStore.put(b, "1".getBytes()); @@ -437,7 +447,9 @@ public void shouldForwardChangedValuesDuringFlush() { new KeyValueTimestamp<>( bDeserialized, new Change<>("1", null), - DEFAULT_TIMESTAMP)), + DEFAULT_TIMESTAMP + ) + ), flushListener.forwarded ); flushListener.forwarded.clear(); @@ -450,7 +462,9 @@ public void shouldForwardChangedValuesDuringFlush() { new KeyValueTimestamp<>( aDeserialized, new Change<>("1", null), - DEFAULT_TIMESTAMP)), + DEFAULT_TIMESTAMP + ) + ), flushListener.forwarded ); flushListener.forwarded.clear(); @@ -463,7 +477,9 @@ public void shouldForwardChangedValuesDuringFlush() { new KeyValueTimestamp<>( aDeserialized, new Change<>("2", "1"), - DEFAULT_TIMESTAMP)), + DEFAULT_TIMESTAMP + ) + ), flushListener.forwarded ); flushListener.forwarded.clear(); @@ -476,7 +492,9 @@ public void shouldForwardChangedValuesDuringFlush() { new KeyValueTimestamp<>( aDeserialized, new Change<>(null, "2"), - DEFAULT_TIMESTAMP)), + DEFAULT_TIMESTAMP + ) + ), flushListener.forwarded ); flushListener.forwarded.clear(); @@ -513,18 +531,23 @@ public void shouldNotForwardChangedValuesDuringFlushWhenSendOldValuesDisabled() cachingStore.flush(); assertEquals( - asList(new KeyValueTimestamp<>( + asList( + new KeyValueTimestamp<>( aDeserialized, new Change<>("1", null), - DEFAULT_TIMESTAMP), + DEFAULT_TIMESTAMP + ), new KeyValueTimestamp<>( aDeserialized, new Change<>("2", null), - DEFAULT_TIMESTAMP), + DEFAULT_TIMESTAMP + ), new KeyValueTimestamp<>( aDeserialized, new Change<>(null, null), - DEFAULT_TIMESTAMP)), + DEFAULT_TIMESTAMP + ) + ), flushListener.forwarded ); flushListener.forwarded.clear(); @@ -548,8 +571,10 @@ public void shouldReturnSameResultsForSingleKeyFindSessionsAndEqualKeyRangeFindS cachingStore.put(new Windowed<>(keyAA, new SessionWindow(4, 5)), "3".getBytes()); cachingStore.put(new Windowed<>(keyB, new SessionWindow(6, 7)), "4".getBytes()); - final KeyValueIterator, byte[]> singleKeyIterator = cachingStore.findSessions(keyAA, 0L, 10L); - final KeyValueIterator, byte[]> keyRangeIterator = cachingStore.findSessions(keyAA, keyAA, 0L, 10L); + final KeyValueIterator, byte[]> singleKeyIterator = + cachingStore.findSessions(keyAA, 0L, 10L); + final KeyValueIterator, byte[]> keyRangeIterator = + cachingStore.findSessions(keyAA, keyAA, 0L, 10L); assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); assertEquals(singleKeyIterator.next(), keyRangeIterator.next()); @@ -654,16 +679,19 @@ public void shouldNotThrowInvalidRangeExceptionWhenBackwardWithNegativeFromKey() final Bytes keyTo = Bytes.wrap(Serdes.Integer().serializer().serialize("", 1)); try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(CachingSessionStore.class)) { - final KeyValueIterator, byte[]> iterator = cachingStore.backwardFindSessions(keyFrom, keyTo, 0L, 10L); + final KeyValueIterator, byte[]> iterator = + cachingStore.backwardFindSessions(keyFrom, keyTo, 0L, 10L); assertFalse(iterator.hasNext()); final List messages = appender.getMessages(); assertThat( messages, - hasItem("Returning empty iterator for fetch with invalid key range: from > to." + - " This may be due to range arguments set in the wrong order, " + - "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + - " Note that the built-in numerical serdes do not follow this for negative numbers") + hasItem( + "Returning empty iterator for fetch with invalid key range: from > to." + + " This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + + " Note that the built-in numerical serdes do not follow this for negative numbers" + ) ); } } @@ -680,10 +708,12 @@ public void shouldNotThrowInvalidRangeExceptionWithNegativeFromKey() { final List messages = appender.getMessages(); assertThat( messages, - hasItem("Returning empty iterator for fetch with invalid key range: from > to." + - " This may be due to range arguments set in the wrong order, " + - "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + - " Note that the built-in numerical serdes do not follow this for negative numbers") + hasItem( + "Returning empty iterator for fetch with invalid key range: from > to." + + " This may be due to range arguments set in the wrong order, " + + "or serdes that don't preserve ordering when lexicographically comparing the serialized bytes." + + " Note that the built-in numerical serdes do not follow this for negative numbers" + ) ); } } @@ -707,9 +737,9 @@ private void addSingleSession(final String sessionId, final List implements CacheFlushListener { - final Deserializer keyDeserializer; - final Deserializer valueDesializer; - final List>> forwarded = new LinkedList<>(); + private final Deserializer keyDeserializer; + private final Deserializer valueDesializer; + private final List>> forwarded = new LinkedList<>(); CacheFlushListenerStub(final Deserializer keyDeserializer, final Deserializer valueDesializer) { @@ -728,7 +758,9 @@ public void apply(final byte[] key, new Change<>( valueDesializer.deserialize(null, newValue), valueDesializer.deserialize(null, oldValue)), - timestamp)); + timestamp + ) + ); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java index 1ff3153813a8f..8edda96ef442f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSessionBytesStoreTest.java @@ -202,4 +202,6 @@ public void shouldCloseUnderlyingStore() { store.close(); EasyMock.verify(inner); } + + } \ No newline at end of file diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java index d7adca45fb989..4bd125a2cb114 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWrappedSessionStoreIteratorTest.java @@ -45,7 +45,7 @@ public long segmentId(final Bytes key) { private final SessionWindow storeWindow = new SessionWindow(0, 1); private final Iterator, byte[]>> storeKvs = Collections.singleton( - KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get())).iterator(); + KeyValue.pair(new Windowed<>(storeKey, storeWindow), storeKey.get())).iterator(); private final SessionWindow cacheWindow = new SessionWindow(10, 20); private final Iterator> cacheKvs = Collections.singleton( KeyValue.pair( diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java index 5f48c90c36c1c..084e06d89db40 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java @@ -250,8 +250,8 @@ public void shouldWriteBytesToInnerStoreAndRecordPutMetric() { @Test public void shouldFindSessionsFromStoreAndRecordFetchMetric() { expect(innerStore.findSessions(KEY_BYTES, 0, 0)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.findSessions(KEY, 0, 0); @@ -267,8 +267,11 @@ public void shouldFindSessionsFromStoreAndRecordFetchMetric() { @Test public void shouldBackwardFindSessionsFromStoreAndRecordFetchMetric() { expect(innerStore.backwardFindSessions(KEY_BYTES, 0, 0)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn( + new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator() + ) + ); init(); final KeyValueIterator, String> iterator = store.backwardFindSessions(KEY, 0, 0); @@ -284,8 +287,8 @@ public void shouldBackwardFindSessionsFromStoreAndRecordFetchMetric() { @Test public void shouldFindSessionRangeFromStoreAndRecordFetchMetric() { expect(innerStore.findSessions(KEY_BYTES, KEY_BYTES, 0, 0)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.findSessions(KEY, KEY, 0, 0); @@ -301,8 +304,11 @@ public void shouldFindSessionRangeFromStoreAndRecordFetchMetric() { @Test public void shouldBackwardFindSessionRangeFromStoreAndRecordFetchMetric() { expect(innerStore.backwardFindSessions(KEY_BYTES, KEY_BYTES, 0, 0)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn( + new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator() + ) + ); init(); final KeyValueIterator, String> iterator = store.backwardFindSessions(KEY, KEY, 0, 0); @@ -332,8 +338,8 @@ public void shouldRemoveFromStoreAndRecordRemoveMetric() { @Test public void shouldFetchForKeyAndRecordFetchMetric() { expect(innerStore.fetch(KEY_BYTES)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.fetch(KEY); @@ -349,8 +355,11 @@ public void shouldFetchForKeyAndRecordFetchMetric() { @Test public void shouldBackwardFetchForKeyAndRecordFetchMetric() { expect(innerStore.backwardFetch(KEY_BYTES)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn( + new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator() + ) + ); init(); final KeyValueIterator, String> iterator = store.backwardFetch(KEY); @@ -366,8 +375,8 @@ public void shouldBackwardFetchForKeyAndRecordFetchMetric() { @Test public void shouldFetchRangeFromStoreAndRecordFetchMetric() { expect(innerStore.fetch(KEY_BYTES, KEY_BYTES)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn(new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); init(); final KeyValueIterator, String> iterator = store.fetch(KEY, KEY); @@ -383,8 +392,11 @@ public void shouldFetchRangeFromStoreAndRecordFetchMetric() { @Test public void shouldBackwardFetchRangeFromStoreAndRecordFetchMetric() { expect(innerStore.backwardFetch(KEY_BYTES, KEY_BYTES)) - .andReturn(new KeyValueIteratorStub<>( - Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator())); + .andReturn( + new KeyValueIteratorStub<>( + Collections.singleton(KeyValue.pair(WINDOWED_KEY_BYTES, VALUE_BYTES)).iterator() + ) + ); init(); final KeyValueIterator, String> iterator = store.backwardFetch(KEY, KEY); diff --git a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java index 0c8b27630c0da..d604d0859cd0b 100644 --- a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java +++ b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java @@ -84,7 +84,7 @@ public KeyValueIterator, V> backwardFetch(K key) { throw new InvalidStateStoreException("not open"); } if (!sessions.containsKey(key)) { - return new KeyValueIteratorStub<>(Collections., V>>emptyIterator()); + return new KeyValueIteratorStub<>(Collections.emptyIterator()); } return new KeyValueIteratorStub<>(sessions.descendingMap().get(key).iterator()); } @@ -97,7 +97,7 @@ public KeyValueIterator, V> fetch(final K from, final K to) { if (sessions.subMap(from, true, to, true).isEmpty()) { return new KeyValueIteratorStub<>(Collections., V>>emptyIterator()); } - final Iterator, V>>> keysIterator = sessions.subMap(from, true, to, true).values().iterator(); + final Iterator, V>>> keysIterator = sessions.subMap(from, true, to, true).values().iterator(); return new KeyValueIteratorStub<>( new Iterator, V>>() { @@ -118,6 +118,7 @@ public boolean hasNext() { public KeyValue, V> next() { return it.next(); } + } ); } @@ -128,10 +129,10 @@ public KeyValueIterator, V> backwardFetch(K from, K to) { throw new InvalidStateStoreException("not open"); } if (sessions.subMap(from, true, to, true).isEmpty()) { - return new KeyValueIteratorStub<>(Collections., V>>emptyIterator()); + return new KeyValueIteratorStub<>(Collections.emptyIterator()); } - final Iterator, V>>> keysIterator = sessions.subMap(from, true, to, true) - .descendingMap().values().iterator(); + final Iterator, V>>> keysIterator = + sessions.subMap(from, true, to, true).descendingMap().values().iterator(); return new KeyValueIteratorStub<>( new Iterator, V>>() { From 19324130d62a482b001c0e85b8fe51a0160d10fd Mon Sep 17 00:00:00 2001 From: John Roesler Date: Wed, 7 Oct 2020 22:41:58 -0500 Subject: [PATCH 13/14] fix formatting --- .../state/internals/MeteredSessionStoreTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java index 084e06d89db40..bd55166e0a38d 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java @@ -464,8 +464,7 @@ public void shouldThrowNullPointerOnFindSessionsRangeIfToIsNull() { store.findSessions("a", null, 0, 0); } - private interface CachedSessionStore extends SessionStore, CachedStateStore { - } + private interface CachedSessionStore extends SessionStore, CachedStateStore { } @SuppressWarnings("unchecked") @Test @@ -522,9 +521,9 @@ private KafkaMetric metric(final String name) { private List storeMetrics() { return metrics.metrics() - .keySet() - .stream() - .filter(name -> name.group().equals(storeLevelGroup) && name.tags().equals(tags)) - .collect(Collectors.toList()); + .keySet() + .stream() + .filter(name -> name.group().equals(storeLevelGroup) && name.tags().equals(tags)) + .collect(Collectors.toList()); } } From 14fce521346b290407c971e42bd684f55b512785 Mon Sep 17 00:00:00 2001 From: John Roesler Date: Wed, 7 Oct 2020 23:17:59 -0500 Subject: [PATCH 14/14] fix conflict with trunk --- .../streams/state/internals/CachingInMemorySessionStoreTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java index 8775072581e4b..e584e2ca706b6 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemorySessionStoreTest.java @@ -78,6 +78,7 @@ public class CachingInMemorySessionStoreTest { private final Bytes keyB = Bytes.wrap("b".getBytes()); private SessionStore underlyingStore; + private InternalMockProcessorContext context; private CachingSessionStore cachingStore; private ThreadCache cache;