From 9365f34f9d0680bf2f7c99fe8b9645627472b669 Mon Sep 17 00:00:00 2001 From: Rushabh Shah Date: Fri, 19 Nov 2021 10:47:29 -0500 Subject: [PATCH 1/7] HBASE-26468 Region Server doesn't exit cleanly incase it crashes. --- .../java/org/apache/hadoop/hbase/util/ServerCommandLine.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java index f99a0908cd6a..eb2be39c7fdb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -147,9 +147,7 @@ public static void logProcessInfo(Configuration conf) { public void doMain(String args[]) { try { int ret = ToolRunner.run(HBaseConfiguration.create(), this, args); - if (ret != 0) { - System.exit(ret); - } + System.exit(ret); } catch (Exception e) { LOG.error("Failed to run", e); System.exit(-1); From bbeba1fa78abb9131ee65ba84f4e4857f44922f4 Mon Sep 17 00:00:00 2001 From: Rushabh Shah Date: Sat, 20 Nov 2021 09:01:15 -0500 Subject: [PATCH 2/7] Address review comments --- .../java/org/apache/hadoop/hbase/util/ServerCommandLine.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java index eb2be39c7fdb..561482c00e7e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -141,8 +141,8 @@ public static void logProcessInfo(Configuration conf) { } /** - * Parse and run the given command line. This may exit the JVM if - * a nonzero exit code is returned from run(). + * Parse and run the given command line. This will exit the JVM with + * the exit code returned from run(). */ public void doMain(String args[]) { try { From 82883ba9b7edcf14100a2b8688dcac27f403dec7 Mon Sep 17 00:00:00 2001 From: Rushabh Shah Date: Thu, 25 Nov 2021 09:59:58 -0500 Subject: [PATCH 3/7] Addressing review comments --- .../hadoop/hbase/util/ServerCommandLine.java | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java index 561482c00e7e..d8c2e5d65dff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -25,6 +25,7 @@ import java.util.Locale; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.conf.Configuration; @@ -143,14 +144,49 @@ public static void logProcessInfo(Configuration conf) { /** * Parse and run the given command line. This will exit the JVM with * the exit code returned from run(). + * If return code is 0, wait for atmost 30 seconds for all non-daemon threads to quit, + * otherwise exit the jvm */ public void doMain(String args[]) { try { int ret = ToolRunner.run(HBaseConfiguration.create(), this, args); - System.exit(ret); + if (ret != 0) { + System.exit(ret); + } + // Return code is 0 here. + boolean forceStop = false; + long now = EnvironmentEdgeManager.currentTime(); + while (isNonDaemonThreadRunning()) { + if (EnvironmentEdgeManager.currentTime() - now > 30 * 1000) { + forceStop = true; + break; + } + Thread.sleep(1000); + } + if (forceStop) { + LOG.error("Failed to stop all non-daemon threads, so force quitting"); + System.exit(-1); + } } catch (Exception e) { LOG.error("Failed to run", e); System.exit(-1); } } + + /** + * Checks whether any non-daemon thread is running. + * @return true if there are non daemon threads running, otherwise false + */ + public boolean isNonDaemonThreadRunning() { + AtomicInteger nonDaemonThreadCount = new AtomicInteger(); + Set threads = Thread.getAllStackTraces().keySet(); + threads.forEach(t -> { + // Exclude current thread + if (t.getId() != Thread.currentThread().getId() && !t.isDaemon()) { + nonDaemonThreadCount.getAndIncrement(); + LOG.info("Non daemon thread name: " + t.getName() + " still alive"); + } + }); + return nonDaemonThreadCount.get() > 0; + } } From b9856815407dc0b509ff45a180b278f087b84841 Mon Sep 17 00:00:00 2001 From: Rushabh Shah Date: Fri, 26 Nov 2021 14:14:05 -0500 Subject: [PATCH 4/7] Addressing review comments --- .../org/apache/hadoop/hbase/util/Threads.java | 32 +++++++++++++++++++ .../hadoop/hbase/util/ServerCommandLine.java | 23 ++----------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java index 610fe05214e4..04784fdc8c79 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java @@ -20,10 +20,12 @@ import java.io.PrintStream; import java.lang.Thread.UncaughtExceptionHandler; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.StringUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -197,4 +199,34 @@ public static void setLoggingUncaughtExceptionHandler(Thread t) { public static void printThreadInfo(PrintStream stream, String title) { ReflectionUtils.printThreadInfo(stream, title); } + + /** + * Checks whether any non-daemon thread is running. + * @return true if there are non daemon threads running, otherwise false + */ + public static boolean isNonDaemonThreadRunning() { + AtomicInteger nonDaemonThreadCount = new AtomicInteger(); + Set threads = Thread.getAllStackTraces().keySet(); + threads.forEach(t -> { + // Exclude current thread + if (t.getId() != Thread.currentThread().getId() && !t.isDaemon()) { + nonDaemonThreadCount.getAndIncrement(); + LOG.info("Non daemon thread {} is still alive", t.getName()); + LOG.info(printStackTrace(t)); + } + }); + return nonDaemonThreadCount.get() > 0; + } + + /* + Print stack trace of the passed thread + */ + public static String printStackTrace(Thread t) { + StringBuilder sb = new StringBuilder(); + for (StackTraceElement frame: t.getStackTrace()) { + sb.append("\n"); + sb.append(" " + frame.toString()); + } + return sb.toString(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java index d8c2e5d65dff..de9887b35eb7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -25,7 +25,6 @@ import java.util.Locale; import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.conf.Configuration; @@ -37,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hbase.util.Threads.isNonDaemonThreadRunning; /** * Base class for command lines that start up various HBase daemons. @@ -155,9 +155,9 @@ public void doMain(String args[]) { } // Return code is 0 here. boolean forceStop = false; - long now = EnvironmentEdgeManager.currentTime(); + long startTime = EnvironmentEdgeManager.currentTime(); while (isNonDaemonThreadRunning()) { - if (EnvironmentEdgeManager.currentTime() - now > 30 * 1000) { + if (EnvironmentEdgeManager.currentTime() - startTime > 30 * 1000) { forceStop = true; break; } @@ -172,21 +172,4 @@ public void doMain(String args[]) { System.exit(-1); } } - - /** - * Checks whether any non-daemon thread is running. - * @return true if there are non daemon threads running, otherwise false - */ - public boolean isNonDaemonThreadRunning() { - AtomicInteger nonDaemonThreadCount = new AtomicInteger(); - Set threads = Thread.getAllStackTraces().keySet(); - threads.forEach(t -> { - // Exclude current thread - if (t.getId() != Thread.currentThread().getId() && !t.isDaemon()) { - nonDaemonThreadCount.getAndIncrement(); - LOG.info("Non daemon thread name: " + t.getName() + " still alive"); - } - }); - return nonDaemonThreadCount.get() > 0; - } } From fdf214aab17640adf003ac2b2264bfd47c3c1155 Mon Sep 17 00:00:00 2001 From: Rushabh Shah Date: Mon, 29 Nov 2021 15:17:39 -0500 Subject: [PATCH 5/7] Addressing review comments --- .../src/main/java/org/apache/hadoop/hbase/util/Threads.java | 3 +-- .../java/org/apache/hadoop/hbase/util/ServerCommandLine.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java index 04784fdc8c79..a0f0bdc0826b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java @@ -224,8 +224,7 @@ public static boolean isNonDaemonThreadRunning() { public static String printStackTrace(Thread t) { StringBuilder sb = new StringBuilder(); for (StackTraceElement frame: t.getStackTrace()) { - sb.append("\n"); - sb.append(" " + frame.toString()); + sb.append("\n").append(" " + frame.toString()); } return sb.toString(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java index de9887b35eb7..731becaf7896 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -164,7 +164,7 @@ public void doMain(String args[]) { Thread.sleep(1000); } if (forceStop) { - LOG.error("Failed to stop all non-daemon threads, so force quitting"); + LOG.error("Failed to stop all non-daemon threads, so terminating JVM "); System.exit(-1); } } catch (Exception e) { From 46f130380f3a223cccd49cddac21a626aa88a52b Mon Sep 17 00:00:00 2001 From: Rushabh Shah Date: Mon, 29 Nov 2021 15:18:21 -0500 Subject: [PATCH 6/7] Removing whitespace --- .../java/org/apache/hadoop/hbase/util/ServerCommandLine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java index 731becaf7896..7f89d59fbca7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -164,7 +164,7 @@ public void doMain(String args[]) { Thread.sleep(1000); } if (forceStop) { - LOG.error("Failed to stop all non-daemon threads, so terminating JVM "); + LOG.error("Failed to stop all non-daemon threads, so terminating JVM"); System.exit(-1); } } catch (Exception e) { From 5aad0284344b8e1d4f5aab13319c1e1c9b85e37c Mon Sep 17 00:00:00 2001 From: Rushabh Shah Date: Tue, 30 Nov 2021 10:35:24 -0500 Subject: [PATCH 7/7] Addressing review comments --- .../src/main/java/org/apache/hadoop/hbase/util/Threads.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java index a0f0bdc0826b..d80639ea68c6 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java @@ -224,7 +224,7 @@ public static boolean isNonDaemonThreadRunning() { public static String printStackTrace(Thread t) { StringBuilder sb = new StringBuilder(); for (StackTraceElement frame: t.getStackTrace()) { - sb.append("\n").append(" " + frame.toString()); + sb.append("\n").append(" ").append(frame.toString()); } return sb.toString(); }