Skip to content

Commit

Permalink
Issue 3206: Flaky-test: BookieZKExpireTest.testBookieServerZKExpireBe…
Browse files Browse the repository at this point in the history 
…haviour (#3415)

* Issue 3206: Flaky-test: BookieZKExpireTest.testBookieServerZKExpireBehaviour

* clear property at the end of the test
  • Loading branch information
@dlg99
dlg99 authored Jul 21, 2022
1 parent 6b03b01 commit 0b910db
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 11 deletions.
10 changes: 6 additions & 4 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,11 @@ public BookieStateManager(ServerConfiguration conf,
this.rm = rm;
if (this.rm != null) {
rm.addRegistrationListener(() -> {
forceToUnregistered();
// schedule a re-register operation
registerBookie(false);
});
log.info("Trying to re-register the bookie");
forceToUnregistered();
// schedule a re-register operation
registerBookie(false);
});
}

this.statusDirs = statusDirs;
Expand Down Expand Up @@ -228,6 +229,7 @@ public Future<Void> registerBookie(final boolean throwException) {
@Override
public Void call() throws IOException {
try {
log.info("Re-registering the bookie");
doRegisterBookie();
} catch (IOException ioe) {
if (throwException) {
Expand Down
95 changes: 88 additions & 7 deletions bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

package org.apache.bookkeeper.test;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
Expand All @@ -30,6 +31,7 @@
import java.io.File;
import java.util.HashSet;

import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.bookie.MockUncleanShutdownDetection;
import org.apache.bookkeeper.bookie.TestBookieImpl;
import org.apache.bookkeeper.conf.ServerConfiguration;
Expand All @@ -41,18 +43,93 @@
/**
* Test bookie expiration.
*/
@Slf4j
public class BookieZKExpireTest extends BookKeeperClusterTestCase {

public BookieZKExpireTest() {
super(0);
// 6000 is minimum due to default tick time
baseConf.setZkTimeout(6000);
baseClientConf.setZkTimeout(6000);
}

/*
Should recover from request timeout.
*/
@Test
@SuppressWarnings("deprecation")
public void testBookieServerZKRequestTimeoutBehaviour() throws Exception {
// 6000 is minimum due to default tick time
System.setProperty("zookeeper.request.timeout", "6000");
baseConf.setZkTimeout(24000);
baseClientConf.setZkTimeout(24000);
BookieServer server = null;
try {
File f = tmpDirs.createNew("bookieserver", "test");

HashSet<Thread> threadset = new HashSet<Thread>();
int threadCount = Thread.activeCount();
Thread[] threads = new Thread[threadCount * 2];
threadCount = Thread.enumerate(threads);
for (int i = 0; i < threadCount; i++) {
if (threads[i].getName().indexOf("SendThread") != -1) {
threadset.add(threads[i]);
}
}

ServerConfiguration conf = newServerConfiguration(PortManager.nextFreePort(), f, new File[] { f });
server = new BookieServer(
conf, new TestBookieImpl(conf),
NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
new MockUncleanShutdownDetection());
server.start();

int secondsToWait = 5;
while (!server.isRunning()) {
Thread.sleep(1000);
if (secondsToWait-- <= 0) {
fail("Bookie never started");
}
}
Thread sendthread = null;
threadCount = Thread.activeCount();
threads = new Thread[threadCount * 2];
threadCount = Thread.enumerate(threads);
for (int i = 0; i < threadCount; i++) {
if (threads[i].getName().indexOf("SendThread") != -1
&& !threadset.contains(threads[i])) {
sendthread = threads[i];
break;
}
}
assertNotNull("Send thread not found", sendthread);

log.info("Suspending threads");
sendthread.suspend();
Thread.sleep(12000);
log.info("Resuming threads");
sendthread.resume();

// allow watcher thread to run
Thread.sleep(3000);
assertTrue("Bookie should not shutdown on zk timeout", server.isBookieRunning());
assertTrue("Bookie Server should not shutdown on zk timeout", server.isRunning());
} finally {
System.clearProperty("zookeeper.request.timeout");
server.shutdown();
}
}

/*
Bookie cannot recover from ZK Client's SessionExpired error.
In this case the ZK client must be recreated, reconnect does not work.
Attempt to reconnect by BookieStateManager's RegistrationManager listener
will fail (even if retry it many times).
*/
@Test
public void testBookieServerZKExpireBehaviour() throws Exception {
@SuppressWarnings("deprecation")
public void testBookieServerZKSessionExpireBehaviour() throws Exception {
// 6000 is minimum due to default tick time
System.setProperty("zookeeper.request.timeout", "0");
baseConf.setZkTimeout(6000);
baseClientConf.setZkTimeout(6000);
BookieServer server = null;
try {
File f = tmpDirs.createNew("bookieserver", "test");
Expand Down Expand Up @@ -94,16 +171,20 @@ conf, new TestBookieImpl(conf),
}
assertNotNull("Send thread not found", sendthread);

log.info("Suspending threads");
sendthread.suspend();
Thread.sleep(2 * conf.getZkTimeout());
Thread.sleep(2L * conf.getZkTimeout());
log.info("Resuming threads");
sendthread.resume();

// allow watcher thread to run
Thread.sleep(3000);
assertTrue("Bookie should not shutdown on losing zk session", server.isBookieRunning());
assertTrue("Bookie Server should not shutdown on losing zk session", server.isRunning());
assertFalse("Bookie should shutdown on losing zk session", server.isBookieRunning());
assertFalse("Bookie Server should shutdown on losing zk session", server.isRunning());
} finally {
System.clearProperty("zookeeper.request.timeout");
server.shutdown();
}
}

}

0 comments on commit 0b910db

Please sign in to comment.