-
Notifications
You must be signed in to change notification settings - Fork 3.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Issue 937: add CommandGetLastMessageId to make reader know the end of topic #1066
Changes from all commits
f28020f
928e2e7
ec82b4d
52c4d5b
44b1280
57ddf8a
0ef21df
2f90b85
8da7703
a5bd454
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,7 +19,11 @@ | |
package org.apache.pulsar.client.api; | ||
|
||
import static org.testng.Assert.assertEquals; | ||
import static org.testng.Assert.assertFalse; | ||
import static org.testng.Assert.assertNull; | ||
import static org.testng.Assert.assertTrue; | ||
|
||
import com.google.common.collect.Sets; | ||
import java.io.IOException; | ||
import java.nio.file.Files; | ||
import java.nio.file.Paths; | ||
|
@@ -28,8 +32,8 @@ | |
import java.util.Map; | ||
import java.util.Set; | ||
import java.util.concurrent.TimeUnit; | ||
|
||
import org.apache.pulsar.client.impl.BatchMessageIdImpl; | ||
import org.apache.pulsar.client.impl.MessageImpl; | ||
import org.apache.pulsar.common.policies.data.PersistentTopicStats; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
|
@@ -38,8 +42,6 @@ | |
import org.testng.annotations.BeforeMethod; | ||
import org.testng.annotations.Test; | ||
|
||
import com.google.common.collect.Sets; | ||
|
||
public class TopicReaderTest extends ProducerConsumerBase { | ||
private static final Logger log = LoggerFactory.getLogger(TopicReaderTest.class); | ||
|
||
|
@@ -359,4 +361,103 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMe | |
reader.close(); | ||
log.info("-- Exiting {} test --", methodName); | ||
} | ||
|
||
|
||
@Test | ||
public void testSimpleReaderReachEndofTopic() throws Exception { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. One scenario in which we need to pay attention is when batches are used. With batches, multiple messages are stored in a single BK entry, and the broker treat a batch as a unit, which the consumer will finally break up, presenting the individual messages to application. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. thanks, will add the test. |
||
ReaderConfiguration conf = new ReaderConfiguration(); | ||
Reader reader = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", MessageId.earliest, | ||
conf); | ||
ProducerConfiguration producerConf = new ProducerConfiguration(); | ||
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf); | ||
|
||
// no data write, should return false | ||
assertFalse(reader.hasMessageAvailable()); | ||
|
||
// produce message 0 -- 99 | ||
for (int i = 0; i < 100; i++) { | ||
String message = "my-message-" + i; | ||
producer.send(message.getBytes()); | ||
} | ||
|
||
MessageImpl msg = null; | ||
Set<String> messageSet = Sets.newHashSet(); | ||
int index = 0; | ||
|
||
// read message till end. | ||
while (reader.hasMessageAvailable()) { | ||
msg = (MessageImpl) reader.readNext(1, TimeUnit.SECONDS); | ||
String receivedMessage = new String(msg.getData()); | ||
log.debug("Received message: [{}]", receivedMessage); | ||
String expectedMessage = "my-message-" + (index ++); | ||
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); | ||
} | ||
|
||
assertEquals(index, 100); | ||
// readNext should return null, after reach the end of topic. | ||
assertNull(reader.readNext(1, TimeUnit.SECONDS)); | ||
|
||
// produce message again. | ||
for (int i = 100; i < 200; i++) { | ||
String message = "my-message-" + i; | ||
producer.send(message.getBytes()); | ||
} | ||
|
||
// read message till end again. | ||
while (reader.hasMessageAvailable()) { | ||
msg = (MessageImpl) reader.readNext(1, TimeUnit.SECONDS); | ||
String receivedMessage = new String(msg.getData()); | ||
log.debug("Received message: [{}]", receivedMessage); | ||
String expectedMessage = "my-message-" + (index ++); | ||
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); | ||
} | ||
|
||
assertEquals(index, 200); | ||
// readNext should return null, after reach the end of topic. | ||
assertNull(reader.readNext(1, TimeUnit.SECONDS)); | ||
|
||
producer.close(); | ||
} | ||
|
||
@Test | ||
public void testReaderReachEndofTopicOnMessageWithBatches() throws Exception { | ||
Reader reader = pulsarClient.createReader( | ||
"persistent://my-property/use/my-ns/testReaderReachEndofTopicOnMessageWithBatches", MessageId.earliest, | ||
new ReaderConfiguration()); | ||
|
||
ProducerConfiguration producerConf = new ProducerConfiguration(); | ||
producerConf.setBatchingEnabled(true); | ||
producerConf.setBatchingMaxPublishDelay(100, TimeUnit.MILLISECONDS); | ||
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/testReaderReachEndofTopicOnMessageWithBatches", producerConf); | ||
|
||
// no data write, should return false | ||
assertFalse(reader.hasMessageAvailable()); | ||
|
||
for (int i = 0; i < 100; i++) { | ||
String message = "my-message-" + i; | ||
producer.sendAsync(message.getBytes()); | ||
} | ||
|
||
// Write one sync message to ensure everything before got persistend | ||
producer.send("my-message-10".getBytes()); | ||
|
||
MessageId lastMessageId = null; | ||
int index = 0; | ||
assertTrue(reader.hasMessageAvailable()); | ||
|
||
if (reader.hasMessageAvailable()) { | ||
Message msg = reader.readNext(); | ||
lastMessageId = msg.getMessageId(); | ||
assertEquals(lastMessageId.getClass(), BatchMessageIdImpl.class); | ||
|
||
while (msg != null) { | ||
index++; | ||
msg = reader.readNext(100, TimeUnit.MILLISECONDS); | ||
} | ||
assertEquals(index, 101); | ||
} | ||
|
||
assertFalse(reader.hasMessageAvailable()); | ||
producer.close(); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This should be noted in the javadoc for the client api.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks. will change it.