Merge fixes from OD-1.1 branch

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java

@@ -18,12 +18,11 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Queue;
-import java.util.HashSet;
 import java.util.concurrent.ThreadPoolExecutor;
 
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.service.MasterService;
@@ -35,15 +34,15 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterMetricValues;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.ThreadIDUtil;
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector;
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode;
 
 @SuppressWarnings("unchecked")
 public class MasterServiceEventMetrics extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor {
     public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(
             MasterServiceEventMetrics.class).samplingInterval;
     private static final Logger LOG = LogManager.getLogger(MasterServiceEventMetrics.class);
+    private static final String MASTER_NODE_NOT_UP_METRIC = "MasterNodeNotUp";
     private long lastTaskInsertionOrder;
     private static final int KEYS_PATH_LENGTH = 3;
     private StringBuilder value;
@@ -193,8 +192,12 @@ Queue<Runnable> getMasterServiceCurrentQueue() throws Exception {
                                 (PrioritizedEsThreadPoolExecutor) getMasterServiceTPExecutorField().get(masterService);
                     }
 
-                    masterServiceCurrentQueue =
-                            (Queue<Runnable>) getPrioritizedTPExecutorCurrentField().get(prioritizedEsThreadPoolExecutor);
+                    if (prioritizedEsThreadPoolExecutor != null) {
+                        masterServiceCurrentQueue =
+                                (Queue<Runnable>) getPrioritizedTPExecutorCurrentField().get(prioritizedEsThreadPoolExecutor);
+                    } else {
+                        StatsCollector.instance().logMetric(MASTER_NODE_NOT_UP_METRIC);
+                    }
                 }
             }
         }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/StatsCollector.java

@@ -15,24 +15,23 @@
 
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors;
 
+import java.io.FileInputStream;
+import java.io.InputStream;
 import java.text.SimpleDateFormat;
 import java.util.Date;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.List;
-import java.util.Vector;
 import java.util.Properties;
-
-import java.io.InputStream;
-import java.io.FileInputStream;
+import java.util.Vector;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import java.util.concurrent.atomic.AtomicInteger;
 
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration;
 import com.google.common.annotations.VisibleForTesting;
 
@@ -75,6 +74,11 @@ public void logException() {
 
     public void logException(StatExceptionCode statExceptionCode) {
         incCounter(statExceptionCode.toString());
+        incErrorCounter();
+    }
+
+    public void logMetric(final String metricName) {
+        incCounter(metricName);
     }
 
     public void logStatsRecord(Map<String, AtomicInteger> counters, Map<String, String> statsdata, 
@@ -133,7 +137,9 @@ private void incCounter(String counterName) {
         if (val != null) {
             val.getAndIncrement();
         }
+    }
 
+    private void incErrorCounter() {
         AtomicInteger all_val = counters.putIfAbsent(StatExceptionCode.TOTAL_ERROR.toString(), new AtomicInteger(1));
         if (all_val != null) {
             all_val.getAndIncrement();

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PluginSettings.java

@@ -65,6 +65,10 @@ public String getMetricsLocation() {
         return metricsLocation;
     }
 
+    public void setMetricsLocation(final String metricsLocation) {
+        this.metricsLocation = metricsLocation;
+    }
+
     public int getMetricsDeletionInterval() {
         return metricsDeletionInterval * 60 * 1000;
     }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java

@@ -27,9 +27,9 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
 
 @SuppressWarnings("checkstyle:constantname")
 public class PerformanceAnalyzerMetrics {
@@ -82,7 +82,8 @@ public static String getCurrentTimeMetric() {
 
     public static String generatePath(long startTime, String... keysPath) {
         Path sDevShmLocationPath = Paths.get(sDevShmLocation)
-                                    .resolve(Paths.get(String.valueOf(PerformanceAnalyzerMetrics.getTimeInterval(startTime)), keysPath));
+                                        .resolve(Paths.get(String.valueOf(
+                                                PerformanceAnalyzerMetrics.getTimeInterval(startTime)), keysPath));
         return sDevShmLocationPath.toString();
     }
 

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CustomMetricsLocationTestBase.java

@@ -0,0 +1,25 @@
+package com.amazon.opendistro.elasticsearch.performanceanalyzer;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
+
+public class CustomMetricsLocationTestBase {
+
+    private static final Path METRICS_LOCATION = Paths.get("build/tmp/junit_metrics");
+
+    @Before
+    public void setUp() throws Exception {
+        if (!Files.exists(METRICS_LOCATION)) {
+            Files.createDirectories(METRICS_LOCATION.getParent());
+            Files.createDirectory(METRICS_LOCATION);
+        }
+
+        PluginSettings.instance().setMetricsLocation(METRICS_LOCATION + File.separator);
+    }
+}

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java

@@ -15,11 +15,14 @@
 
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.action;
 
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.junit.Test;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import static org.junit.Assert.assertEquals;
 
-public class PerformanceAnalyzerActionListenerTests {
+public class PerformanceAnalyzerActionListenerTests extends CustomMetricsLocationTestBase {
 
     @Test
     public void testHttpMetrics() {
@@ -28,31 +31,31 @@ public void testHttpMetrics() {
         PerformanceAnalyzerActionListener performanceanalyzerActionListener = new PerformanceAnalyzerActionListener();
         performanceanalyzerActionListener.saveMetricValues("XYZADFAS", startTimeInMills, "bulk", "bulkId", "start");
         String fetchedValue = PerformanceAnalyzerMetrics.getMetric(
-                PerformanceAnalyzerMetrics.sDevShmLocation
+                PluginSettings.instance().getMetricsLocation()
                         + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/bulk/bulkId/start");
         assertEquals("XYZADFAS", fetchedValue);
 
         String startMetricsValue = performanceanalyzerActionListener.generateStartMetrics(123, "val2", 0).toString();
         performanceanalyzerActionListener.saveMetricValues(startMetricsValue,  startTimeInMills, "search", "searchId1", "start");
         fetchedValue = PerformanceAnalyzerMetrics.getMetric(
-                PerformanceAnalyzerMetrics.sDevShmLocation
+                PluginSettings.instance().getMetricsLocation()
                         + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId1/start");
         assertEquals(startMetricsValue, fetchedValue);
 
         String finishMetricsValue = performanceanalyzerActionListener.generateFinishMetrics(456, 200, "val4").toString();
         performanceanalyzerActionListener.saveMetricValues(finishMetricsValue, startTimeInMills, "search", "searchId1", "finish");
         fetchedValue = PerformanceAnalyzerMetrics.getMetric(
-                PerformanceAnalyzerMetrics.sDevShmLocation
+                PluginSettings.instance().getMetricsLocation()
                         + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId1/finish");
         assertEquals(finishMetricsValue, fetchedValue);
 
         performanceanalyzerActionListener.saveMetricValues(finishMetricsValue, startTimeInMills, "search", "searchId2", "finish");
         fetchedValue = PerformanceAnalyzerMetrics.getMetric(
-                PerformanceAnalyzerMetrics.sDevShmLocation
+                PluginSettings.instance().getMetricsLocation()
                         + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId2/finish");
         assertEquals(finishMetricsValue, fetchedValue);
 
-        PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation 
+        PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation()
                 + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills));
     }
 }

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java

@@ -16,20 +16,23 @@
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors;
 
 import org.junit.Test;
-import static org.junit.Assert.assertEquals;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
+import static org.junit.Assert.assertEquals;
 
-public class CircuitBreakerCollectorTests {
+public class CircuitBreakerCollectorTests extends CustomMetricsLocationTestBase {
 
     @Test
     public void testCircuitBreakerMetrics() {
         System.setProperty("performanceanalyzer.metrics.log.enabled", "False");
         long startTimeInMills = 1153721339;
         CircuitBreakerCollector circuitBreakerCollector = new CircuitBreakerCollector();
         circuitBreakerCollector.saveMetricValues("werjbdsiviewur", startTimeInMills);
-        String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation
+        String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation()
                 + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/circuit_breaker/");
-        PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation
+        PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation()
                  + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills));
         assertEquals("werjbdsiviewur", fetchedValue);
 

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java

@@ -15,12 +15,15 @@
 
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors;
 
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.junit.Test;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class MasterServiceMetricsTests {
+public class MasterServiceMetricsTests extends CustomMetricsLocationTestBase {
 
     @Test
     public void testMasterServiceMetrics() {
@@ -31,9 +34,9 @@ public void testMasterServiceMetrics() {
         masterServiceMetrics.saveMetricValues("master_metrics_value", startTimeInMills, "current", "start");
 
 
-        String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation +
+        String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() +
                 PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/pending_tasks/current/start/");
-        PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation
+        PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation()
                  + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills));
         assertEquals("master_metrics_value", fetchedValue);
 

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsMetricsCollectorTests.java

@@ -15,12 +15,15 @@
 
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors;
 
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.junit.Test;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class NodeStatsMetricsCollectorTests {
+public class NodeStatsMetricsCollectorTests extends CustomMetricsLocationTestBase {
 
     @Test
     public void testNodeStatsMetrics() {
@@ -32,9 +35,9 @@ public void testNodeStatsMetrics() {
 
 
         String fetchedValue = PerformanceAnalyzerMetrics.getMetric(
-                PerformanceAnalyzerMetrics.sDevShmLocation
+                PluginSettings.instance().getMetricsLocation()
                         + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/indices/NodesStatsIndex/55/");
-        PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation
+        PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation()
                  + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills));
         assertEquals("89123.23", fetchedValue);
 

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ThreadPoolMetricsCollectorTests.java

@@ -15,11 +15,14 @@
 
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors;
 
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.junit.Test;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import static org.junit.Assert.assertEquals;
 
-public class ThreadPoolMetricsCollectorTests {
+public class ThreadPoolMetricsCollectorTests extends CustomMetricsLocationTestBase {
 
     @Test
     public void testThreadPoolMetrics() {
@@ -31,8 +34,9 @@ public void testThreadPoolMetrics() {
 
 
         String fetchedValue = PerformanceAnalyzerMetrics.getMetric(
-                PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/thread_pool/");
-        PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation
+                PluginSettings.instance().getMetricsLocation()
+                        + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/thread_pool/");
+        PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation()
                  + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills));
         assertEquals("12321.5464", fetchedValue);