Merge pull request #6486 from luiseufrasio/FISH-6466-health-checker-f…

…alse-positive-stuck-thread-community FISH-6466 : port from Payara-Enterprise 5
payara · Dec 4, 2023 · 87d3b12 · 87d3b12
2 parents 6f78607 + 6370bb7
commit 87d3b12
Show file tree

Hide file tree

Showing 8 changed files with 55 additions and 16 deletions.
diff --git a/...ole-plugin-l10n/src/main/resources/fish/payara/admingui/healthcheck/Strings_ja.properties b/...ole-plugin-l10n/src/main/resources/fish/payara/admingui/healthcheck/Strings_ja.properties
@@ -127,6 +127,8 @@ healthcheck.checker.configuration.stuckThreads.timeThresholdLabel=\u95be\u5024\u
 healthcheck.checker.configuration.stuckThreads.timeThresholdLabelHelpText=\u95be\u5024\u306e\u6642\u9593\u91cf\u3092\u5b9a\u7fa9\u3057\u307e\u3059\u3002
 healthcheck.checker.configuration.stuckThreads.thresholdUnitLabel=\u95be\u5024\u306e\u5358\u4f4d
 healthcheck.checker.configuration.stuckThreads.thresholdUnitLabelHelpText=\u95be\u5024\u306e\u6642\u9593\u306e\u5358\u4f4d\u3092\u5b9a\u7fa9\u3057\u307e\u3059\u3002
+healthcheck.checker.configuration.stuckThreads.blacklistPatternsLabel=\u30D5\u30A3\u30EB\u30BF\u30FC\u30D1\u30BF\u30FC\u30F3
+healthcheck.checker.configuration.stuckThreads.blacklistPatternsLabelHelpText=\u30B9\u30EC\u30C3\u30C9\u3092\u30D5\u30A3\u30EB\u30BF\u30EA\u30F3\u30B0\u3059\u308B\u305F\u3081\u306E\u540D\u524D\u30D1\u30BF\u30FC\u30F3\u306E\u30AB\u30F3\u30DE\u533A\u5207\u308A\u30EA\u30B9\u30C8\u3002\u6B63\u898F\u8868\u73FE\u3092\u53D7\u3051\u307E\u3059\u3002
 
 healthcheck.configuration.availableNotifiers=\u4f7f\u7528\u53ef\u80fd\u306a\u30ce\u30fc\u30c6\u30a3\u30d5\u30a1\u30a4\u30e4\u30fc
 healthcheck.configuration.selectedNotifiers=\u9078\u629e\u3055\u308c\u305f\u30ce\u30fc\u30c6\u30a3\u30d5\u30a1\u30a4\u30e4\u30fc
diff --git a/...ice-console-plugin/src/main/resources/fish/payara/admingui/healthcheck/Strings.properties b/...ice-console-plugin/src/main/resources/fish/payara/admingui/healthcheck/Strings.properties
@@ -144,6 +144,8 @@ healthcheck.checker.configuration.stuckThreads.timeThresholdLabel=Threshold time
 healthcheck.checker.configuration.stuckThreads.timeThresholdLabelHelpText=Defines amount of time for the threshold. 
 healthcheck.checker.configuration.stuckThreads.thresholdUnitLabel=Threshold unit
 healthcheck.checker.configuration.stuckThreads.thresholdUnitLabelHelpText=Defines unit of time for the threshold.
+healthcheck.checker.configuration.stuckThreads.blacklistPatternsLabel=Blacklist Patterns
+healthcheck.checker.configuration.stuckThreads.blacklistPatternsLabelHelpText=A comma separated list of thread name patterns for the stuck thread checker to ignore. Accepts regular expressions.
 
 healthcheck.configuration.availableNotifiers=Avaliable Notifiers
 healthcheck.configuration.selectedNotifiers=Selected Notifiers
diff --git a/...vice-console-plugin/src/main/resources/healthcheck/checkers/stuckThreadsConfiguration.jsf b/...vice-console-plugin/src/main/resources/healthcheck/checkers/stuckThreadsConfiguration.jsf
@@ -1,6 +1,6 @@
 <!--
 
- Copyright (c) [2017-2020] Payara Foundation and/or its affiliates. All rights reserved.
+ Copyright (c) [2017-2023] Payara Foundation and/or its affiliates. All rights reserved.
 
  The contents of this file are subject to the terms of either the GNU
  General Public License Version 2 only ("GPL") or the Common Development
@@ -139,6 +139,11 @@
             <sun:dropDown id="unitDropdown" selected="#{pageSession.valueMap['thresholdUnit']}" 
                           labels={"MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS"} />
         </sun:property>
+        <sun:property id="blacklistPatternsProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
+                      label="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.blacklistPatternsLabel}"
+                      helpText="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.blacklistPatternsLabelHelpText}">
+            <sun:textField id="blacklistPatterns" columns="$int{75}" maxLength="255" text="#{pageSession.valueMap['blacklistPatterns']}" />
+        </sun:property>
     </sun:propertySheetSection>
 </sun:propertySheet>
 </sun:form>

diff --git a/...src/main/java/fish/payara/nucleus/healthcheck/HealthCheckStuckThreadExecutionOptions.java b/...src/main/java/fish/payara/nucleus/healthcheck/HealthCheckStuckThreadExecutionOptions.java
@@ -1,7 +1,7 @@
 /*
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  * 
- *    Copyright (c) [2017-2020] Payara Foundation and/or its affiliates. All rights reserved.
+ *    Copyright (c) [2017-2023] Payara Foundation and/or its affiliates. All rights reserved.
  * 
  *     The contents of this file are subject to the terms of either the GNU
  *     General Public License Version 2 only ("GPL") or the Common Development
@@ -49,12 +49,19 @@ public class HealthCheckStuckThreadExecutionOptions extends HealthCheckExecution
 
     private long timeStuck;
     private TimeUnit unitStuck;
-
-    public HealthCheckStuckThreadExecutionOptions(boolean enabled, long time, TimeUnit unit, boolean addToMicroProfileHealth, 
-            long timeStuck, TimeUnit unitStuck) {
+    private String blacklistPatterns;
+
+    public HealthCheckStuckThreadExecutionOptions(boolean enabled, long time, TimeUnit unit, boolean addToMicroProfileHealth,
+                                                  long timeStuck, TimeUnit unitStuck) {
+        this(enabled, time, unit, addToMicroProfileHealth, timeStuck, unitStuck, "");
+    }
+
+    public HealthCheckStuckThreadExecutionOptions(boolean enabled, long time, TimeUnit unit, boolean addToMicroProfileHealth,
+                                                  long timeStuck, TimeUnit unitStuck, String blacklistPatterns) {
         super(enabled, time, unit, addToMicroProfileHealth);
         this.timeStuck = timeStuck;
         this.unitStuck = unitStuck;
+        this.blacklistPatterns = blacklistPatterns;
     }
 
     /**
@@ -72,5 +79,12 @@ public Long getTimeStuck(){
     public TimeUnit getUnitStuck(){
         return unitStuck;
     }
-
+
+    /**
+     *
+     * @return Comma separated patterns to filter out threads by name
+     */
+    public String getBlacklistPatterns() {
+        return blacklistPatterns;
+    }
 }
diff --git a/...core/src/main/java/fish/payara/nucleus/healthcheck/admin/GetHealthCheckConfiguration.java b/...core/src/main/java/fish/payara/nucleus/healthcheck/admin/GetHealthCheckConfiguration.java
@@ -1,7 +1,7 @@
 /*
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
- * Copyright (c) [2016-2021] Payara Foundation and/or its affiliates. All rights reserved.
+ * Copyright (c) [2016-2023] Payara Foundation and/or its affiliates. All rights reserved.
  *
  * The contents of this file are subject to the terms of either the GNU
  * General Public License Version 2 only ("GPL") or the Common Development
@@ -108,7 +108,7 @@ public class GetHealthCheckConfiguration implements AdminCommand, HealthCheckCon
             "Retry Count"};
     final static String thresholdDiagnosticsHeaders[] = {"Name", "Enabled", "Time", "Unit", "Add to MicroProfile Health", "Critical Threshold",
             "Warning Threshold", "Good Threshold"};
-    final static String stuckThreadsHeaders[] = {"Name", "Enabled", "Time", "Unit", "Add to MicroProfile Health", "Threshold Time", "Threshold Unit"};
+    final static String stuckThreadsHeaders[] = {"Name", "Enabled", "Time", "Unit", "Add to MicroProfile Health", "Threshold Time", "Threshold Unit", "Blacklist Patterns"};
     final static String MPHealthCheckHeaders[] = {"Name", "Enabled", "Time", "Unit", "Add to MicroProfile Health", "Timeout"};
     final static String microProfileMetricsCheckHeaders[] = {"Name", "Enabled", "Time", "Unit", "Add to MicroProfile Health"};
     final static String monitoredMicroProfileMetricHeaders[] = {"Monitored Metric Name", "Description" };
@@ -270,14 +270,15 @@ public void execute(AdminCommandContext context) {
             } else if (checker instanceof StuckThreadsChecker) {
                 StuckThreadsChecker stuckThreadsChecker = (StuckThreadsChecker) checker;
 
-                Object values[] = new Object[7];
+                Object values[] = new Object[8];
                 values[0] = stuckThreadsChecker.getName();
                 values[1] = stuckThreadsChecker.getEnabled();
                 values[2] = stuckThreadsChecker.getTime();
                 values[3] = stuckThreadsChecker.getUnit();
                 values[4] = stuckThreadsChecker.getAddToMicroProfileHealth();
                 values[5] = stuckThreadsChecker.getThreshold();
                 values[6] = stuckThreadsChecker.getThresholdTimeUnit();
+                values[7] = stuckThreadsChecker.getBlacklistPatterns();
                 stuckThreadsColumnFormatter.addRow(values);
 
                 addStuckThreadsCheckerExtrasProps(stuckThreadsExtrasProps, stuckThreadsChecker);
@@ -424,6 +425,7 @@ private void addStuckThreadsCheckerExtrasProps(Properties stuckThreadsExtrasProp
             extraPropsMap.put("threshold", stuckThreadsChecker.getThreshold());
             extraPropsMap.put("thresholdUnit", stuckThreadsChecker.getThresholdTimeUnit());
         }
+        extraPropsMap.put("blacklistPatterns", stuckThreadsChecker.getBlacklistPatterns());
         stuckThreadsExtrasProps.put(stuckThreadsPropertyName, extraPropsMap);
     }
 

diff --git a/...c/main/java/fish/payara/nucleus/healthcheck/admin/SetHealthCheckServiceConfiguration.java b/...c/main/java/fish/payara/nucleus/healthcheck/admin/SetHealthCheckServiceConfiguration.java
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright (c) 2019-2021 Payara Foundation and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019-2023 Payara Foundation and/or its affiliates. All rights reserved.
  *
  * The contents of this file are subject to the terms of either the GNU
  * General Public License Version 2 only ("GPL") or the Common Development
@@ -188,6 +188,9 @@ public class SetHealthCheckServiceConfiguration implements AdminCommand {
             acceptableValues = "DAYS,HOURS,MILLISECONDS,MINUTES,SECONDS")
     private String stuckThreadsThresholdUnit;
 
+    @Param(name = "stuck-threads-blacklist-patterns", alias = "blacklistPatterns", optional = true)
+    private String stuckThreadsBlacklistPatterns;
+
     // threshold properties params:
 
     @Param(name = "threshold-critical", alias = "thresholdCritical", optional = true, 
@@ -383,6 +386,8 @@ private <C extends Checker> Checker updateProperties(Checker config, Class<C> ty
                     stuckThreadsThreshold, StuckThreadsChecker::setThreshold);
             updateProperty(stuckThreadsConfig, "stuck-threads-threshold-unit", stuckThreadsConfig.getThresholdTimeUnit(), 
                     stuckThreadsThresholdUnit, StuckThreadsChecker::setThresholdTimeUnit);
+            updateProperty(stuckThreadsConfig, "stuck-threads-blacklist-patterns",
+                    stuckThreadsConfig.getBlacklistPatterns(), stuckThreadsBlacklistPatterns, StuckThreadsChecker::setBlacklistPatterns);
         }
         if (MicroProfileMetricsChecker.class.isAssignableFrom(type)) {
             MicroProfileMetricsChecker microProfileMetricsConfig = (MicroProfileMetricsChecker) config;           

diff --git a/...core/src/main/java/fish/payara/nucleus/healthcheck/configuration/StuckThreadsChecker.java b/...core/src/main/java/fish/payara/nucleus/healthcheck/configuration/StuckThreadsChecker.java
@@ -1,7 +1,7 @@
 /*
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
- * Copyright (c) [2017-2020] Payara Foundation and/or its affiliates. All rights reserved.
+ * Copyright (c) [2017-2023] Payara Foundation and/or its affiliates. All rights reserved.
  *
  * The contents of this file are subject to the terms of either the GNU
  * General Public License Version 2 only ("GPL") or the Common Development
@@ -63,5 +63,8 @@ public interface StuckThreadsChecker extends Checker {
     @Attribute(defaultValue = "MINUTES")
     String getThresholdTimeUnit();
     void setThresholdTimeUnit(String value) throws PropertyVetoException;
-
+
+    @Attribute(defaultValue = "")
+    String getBlacklistPatterns();
+    void setBlacklistPatterns(String blacklistPatterns);
 }
diff --git a/...ck-stuck/src/main/java/fish/payara/nucleus/healthcheck/stuck/StuckThreadsHealthCheck.java b/...ck-stuck/src/main/java/fish/payara/nucleus/healthcheck/stuck/StuckThreadsHealthCheck.java
@@ -1,7 +1,7 @@
 /*
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
- * Copyright (c) 2016-2021 Payara Foundation and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016-2023 Payara Foundation and/or its affiliates. All rights reserved.
  *
  * The contents of this file are subject to the terms of either the GNU
  * General Public License Version 2 only ("GPL") or the Common Development
@@ -60,6 +60,7 @@
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
 import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -202,19 +203,24 @@ private void acceptStuckThreads(StuckThreadConsumer consumer) {
         long thresholdInMillis = getThresholdInMillis();
         long now = System.currentTimeMillis();
         ConcurrentHashMap<Long, Long> threads = stuckThreadsStore.getThreads();
-        for (Entry<Long, Long> thread : threads.entrySet()){
+        String[] blacklist = checker.getBlacklistPatterns().split(",");
+        for (Entry<Long, Long> thread : threads.entrySet()) {
             Long threadId = thread.getKey();
             long workStartedTime = thread.getValue();
             long timeWorkingInMillis = now - workStartedTime;
             if (timeWorkingInMillis > thresholdInMillis){
                 ThreadInfo info = bean.getThreadInfo(threadId, Integer.MAX_VALUE);
-                if (info != null){ //check thread hasn't died already
+                if (info != null && !isInBlacklist(info.getThreadName(), blacklist)){ //check thread hasn't died already
                     consumer.accept(workStartedTime, timeWorkingInMillis, thresholdInMillis, info);
                 }
             }
         }
     }
 
+    private boolean isInBlacklist(String threadName, String[] blacklistPatterns) {
+        return Arrays.stream(blacklistPatterns).anyMatch(threadName::matches);
+    }
+
     private long getThresholdInMillis() {
         return Math.max(1, TimeUnit.MILLISECONDS.convert(options.getTimeStuck(), options.getUnitStuck()));
     }
@@ -224,7 +230,7 @@ private long getThresholdInMillis() {
     public HealthCheckStuckThreadExecutionOptions constructOptions(StuckThreadsChecker checker) {
         return new HealthCheckStuckThreadExecutionOptions(Boolean.valueOf(checker.getEnabled()),
                 Long.parseLong(checker.getTime()), asTimeUnit(checker.getUnit()), Boolean.valueOf(checker.getAddToMicroProfileHealth()),
-                Long.parseLong(checker.getThreshold()), asTimeUnit(checker.getThresholdTimeUnit()));
+                Long.parseLong(checker.getThreshold()), asTimeUnit(checker.getThresholdTimeUnit()), checker.getBlacklistPatterns());
     }
 
     @Override