Skip to content

Commit

Permalink
HBASE-26468 Region Server doesn't exit cleanly incase it crashes. (#3865
Browse files Browse the repository at this point in the history
) (#3862)

Signed-off-by: Duo Zhang <[email protected]>
Signed-off-by: Geoffrey Jacoby <[email protected]>
Signed-off-by: Viraj Jasani <[email protected]>
  • Loading branch information
shahrs87 authored Dec 1, 2021
1 parent 774484e commit 6fc5bbc
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@

import java.io.PrintStream;
import java.lang.Thread.UncaughtExceptionHandler;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
Expand Down Expand Up @@ -195,4 +197,33 @@ public static void setLoggingUncaughtExceptionHandler(Thread t) {
public static void printThreadInfo(PrintStream stream, String title) {
ReflectionUtils.printThreadInfo(stream, title);
}

/**
* Checks whether any non-daemon thread is running.
* @return true if there are non daemon threads running, otherwise false
*/
public static boolean isNonDaemonThreadRunning() {
AtomicInteger nonDaemonThreadCount = new AtomicInteger();
Set<Thread> threads = Thread.getAllStackTraces().keySet();
threads.forEach(t -> {
// Exclude current thread
if (t.getId() != Thread.currentThread().getId() && !t.isDaemon()) {
nonDaemonThreadCount.getAndIncrement();
LOG.info("Non daemon thread {} is still alive", t.getName());
LOG.info(printStackTrace(t));
}
});
return nonDaemonThreadCount.get() > 0;
}

/*
Print stack trace of the passed thread
*/
public static String printStackTrace(Thread t) {
StringBuilder sb = new StringBuilder();
for (StackTraceElement frame: t.getStackTrace()) {
sb.append("\n").append(" ").append(frame.toString());
}
return sb.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hbase.util.Threads.isNonDaemonThreadRunning;

/**
* Base class for command lines that start up various HBase daemons.
Expand Down Expand Up @@ -141,15 +142,31 @@ public static void logProcessInfo(Configuration conf) {
}

/**
* Parse and run the given command line. This may exit the JVM if
* a nonzero exit code is returned from <code>run()</code>.
* Parse and run the given command line. This will exit the JVM with
* the exit code returned from <code>run()</code>.
* If return code is 0, wait for atmost 30 seconds for all non-daemon threads to quit,
* otherwise exit the jvm
*/
public void doMain(String args[]) {
try {
int ret = ToolRunner.run(HBaseConfiguration.create(), this, args);
if (ret != 0) {
System.exit(ret);
}
// Return code is 0 here.
boolean forceStop = false;
long startTime = EnvironmentEdgeManager.currentTime();
while (isNonDaemonThreadRunning()) {
if (EnvironmentEdgeManager.currentTime() - startTime > 30 * 1000) {
forceStop = true;
break;
}
Thread.sleep(1000);
}
if (forceStop) {
LOG.error("Failed to stop all non-daemon threads, so terminating JVM");
System.exit(-1);
}
} catch (Exception e) {
LOG.error("Failed to run", e);
System.exit(-1);
Expand Down

0 comments on commit 6fc5bbc

Please sign in to comment.