[Collector] Adding Latency and Error service metrics (#442)

* [Collector] Adding Latency and Error service metrics Signed-off-by: Khushboo Rajput <[email protected]> * Removing fastdoubleparser-LICENSE.tx * Removing fastdoubleparser-NOTICE.txt Signed-off-by: Khushboo Rajput <[email protected]> --------- Signed-off-by: Khushboo Rajput <[email protected]>
opensearch-project · May 17, 2023 · 714c59f · 714c59f
1 parent 27ca714
commit 714c59f
Show file tree

Hide file tree

Showing 26 changed files with 125 additions and 79 deletions.
diff --git a/licenses/fastdoubleparser-0.8.0.jar.sha1 b/licenses/fastdoubleparser-0.8.0.jar.sha1
@@ -0,0 +1 @@
+85c25540369921659556ead85e02c99ef0d24280
diff --git a/licenses/fastdoubleparser-LICENSE.txt b/licenses/fastdoubleparser-LICENSE.txt
diff --git a/licenses/fastdoubleparser-NOTICE.txt b/licenses/fastdoubleparser-NOTICE.txt
diff --git a/licenses/jackson-annotations-2.15.0.jar.sha1 b/licenses/jackson-annotations-2.15.0.jar.sha1
@@ -0,0 +1 @@
+89b0fd554928425a776a6e97ed010034312af21d
diff --git a/licenses/jackson-databind-2.15.0.jar.sha1 b/licenses/jackson-databind-2.15.0.jar.sha1
@@ -0,0 +1 @@
+0d41caa3a4e9f85382702a059a65c512f85ac230
diff --git a/licenses/jackson-module-paranamer-2.15.0.jar.sha1 b/licenses/jackson-module-paranamer-2.15.0.jar.sha1
@@ -0,0 +1 @@
+1d92b098577555a9a9e6c8783ce03ae24d49c9d2
diff --git a/licenses/protobuf-java-3.21.12.jar.sha1 b/licenses/protobuf-java-3.21.12.jar.sha1
diff --git a/.../java/org/opensearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java b/.../java/org/opensearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java
@@ -60,7 +60,7 @@ public void collectMetrics(long startTime) {
             return;
         }
 
-        long startTimeMillis = System.currentTimeMillis();
+        long mCurrT = System.currentTimeMillis();
         try {
 
             Method getAdmissionController =
@@ -110,17 +110,15 @@ public void collectMetrics(long startTime) {
             PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
                     WriterMetrics.ADMISSION_CONTROL_COLLECTOR_EXECUTION_TIME,
                     "",
-                    System.currentTimeMillis() - startTimeMillis);
+                    System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR,
-                    getCollectorName(),
-                    System.currentTimeMillis() - startTimeMillis);
             LOG.debug(
                     "Exception in collecting AdmissionControl Metrics: {} for startTime {}",
                     ex::toString,
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR, "", 1);
         }
     }
 
@@ -183,6 +181,8 @@ private boolean canLoadAdmissionControllerClasses() {
                     Class.forName(ADMISSION_CONTROL_SERVICE, false, admissionControlClassLoader);
         } catch (Exception e) {
             LOG.debug("Failed to load AdmissionControllerService classes : {}", e::toString);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR, "", 1);
             return false;
         }
         return true;

diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java
@@ -18,11 +18,13 @@
 import org.opensearch.common.cache.Cache;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.performanceanalyzer.OpenSearchResources;
+import org.opensearch.performanceanalyzer.PerformanceAnalyzerApp;
 import org.opensearch.performanceanalyzer.metrics.AllMetrics.CacheConfigDimension;
 import org.opensearch.performanceanalyzer.metrics.AllMetrics.CacheConfigValue;
 import org.opensearch.performanceanalyzer.metrics.MetricsConfiguration;
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 
 /*
  * Unlike Cache Hit, Miss, Eviction Count and Size, which is tracked on a per shard basis,
@@ -54,6 +56,7 @@ public void collectMetrics(long startTime) {
             return;
         }
 
+        long mCurrT = System.currentTimeMillis();
         value.setLength(0);
         value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds());
         // This is for backward compatibility. Core OpenSearch may or may not emit maxWeight metric.
@@ -117,6 +120,10 @@ public void collectMetrics(long startTime) {
         value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor)
                 .append(shardRequestCacheMaxSizeStatus.serialize());
         saveMetricValues(value.toString(), startTime);
+        PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
+                WriterMetrics.CACHE_CONFIG_METRICS_COLLECTOR_EXECUTION_TIME,
+                "",
+                System.currentTimeMillis() - mCurrT);
     }
 
     @Override

diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollector.java
@@ -68,12 +68,12 @@ public void collectMetrics(long startTime) {
                     System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.CIRCUIT_BREAKER_COLLECTOR_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting CircuitBreaker Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CIRCUIT_BREAKER_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/...va/org/opensearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java b/...va/org/opensearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java
@@ -104,12 +104,12 @@ public void collectMetrics(long startTime) {
                     "",
                     System.currentTimeMillis() - mCurrT);
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting Cluster Applier Service Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/...java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceEventMetrics.java b/...java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceEventMetrics.java
@@ -27,6 +27,7 @@
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.opensearch.performanceanalyzer.metrics.ThreadIDUtil;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
 import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 
 @SuppressWarnings("unchecked")
@@ -83,6 +84,8 @@ public void collectMetrics(long startTime) {
                 return;
             }
 
+            long mCurrT = System.currentTimeMillis();
+
             value.setLength(0);
             Queue<Runnable> current = getClusterManagerServiceCurrentQueue();
 
@@ -145,19 +148,24 @@ public void collectMetrics(long startTime) {
                             PerformanceAnalyzerMetrics.START_FILE_NAME);
 
                     value.setLength(0);
+                    PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
+                            WriterMetrics
+                                    .CLUSTER_MANAGER_SERVICE_EVENTS_METRICS_COLLECTOR_EXECUTION_TIME,
+                            "",
+                            System.currentTimeMillis() - mCurrT);
                 }
             } else {
                 generateFinishMetrics(startTime);
             }
             LOG.debug(() -> "Successfully collected ClusterManager Event Metrics.");
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
-                    WriterMetrics.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting ClusterManager Metrics: {} for startTime {} with ExceptionCode: {}",
                     () -> ex.toString(),
                     () -> startTime,
                     () -> StatExceptionCode.CLUSTER_MANAGER_METRICS_ERROR.toString());
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
         }
     }
 
@@ -235,8 +243,8 @@ Queue<Runnable> getClusterManagerServiceCurrentQueue() throws Exception {
                                         getPrioritizedTPExecutorCurrentField()
                                                 .get(prioritizedOpenSearchThreadPoolExecutor);
                     } else {
-                        PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
-                                WriterMetrics.CLUSTER_MANAGER_NODE_NOT_UP, "", 1);
+                        PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                                ExceptionsAndErrors.CLUSTER_MANAGER_NODE_NOT_UP, "", 1);
                     }
                 }
             }

diff --git a/...main/java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceMetrics.java b/...main/java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceMetrics.java
@@ -19,6 +19,7 @@
 import org.opensearch.performanceanalyzer.metrics.MetricsConfiguration;
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
 import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 
 @SuppressWarnings("unchecked")
@@ -57,6 +58,7 @@ public void collectMetrics(long startTime) {
                 return;
             }
 
+            long mCurrT = System.currentTimeMillis();
             /*
             pendingTasks API returns object of PendingClusterTask which contains insertOrder, priority, source, timeInQueue.
                 Example :
@@ -98,13 +100,17 @@ public void collectMetrics(long startTime) {
                     startTime,
                     PerformanceAnalyzerMetrics.CLUSTER_MANAGER_CURRENT,
                     PerformanceAnalyzerMetrics.CLUSTER_MANAGER_META_DATA);
-        } catch (Exception ex) {
             PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
-                    WriterMetrics.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
+                    WriterMetrics.CLUSTER_MANAGER_SERVICE_METRICS_COLLECTOR_EXECUTION_TIME,
+                    "",
+                    System.currentTimeMillis() - mCurrT);
+        } catch (Exception ex) {
             LOG.debug(
                     "Exception in Collecting ClusterManager Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
         }
     }
 

diff --git a/...g/opensearch/performanceanalyzer/collectors/ClusterManagerThrottlingMetricsCollector.java b/...g/opensearch/performanceanalyzer/collectors/ClusterManagerThrottlingMetricsCollector.java
@@ -85,12 +85,12 @@ public void collectMetrics(long startTime) {
                     System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.CLUSTER_MANAGER_THROTTLING_COLLECTOR_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting ClusterManager Throttling Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_MANAGER_THROTTLING_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/ElectionTermCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/ElectionTermCollector.java
@@ -80,14 +80,12 @@ public void collectMetrics(long startTime) {
                     System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.ELECTION_TERM_COLLECTOR_ERROR,
-                    "",
-                    System.currentTimeMillis() - mCurrT);
             LOG.debug(
                     "Exception in Collecting Election term Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.ELECTION_TERM_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/...in/java/org/opensearch/performanceanalyzer/collectors/FaultDetectionMetricsCollector.java b/...in/java/org/opensearch/performanceanalyzer/collectors/FaultDetectionMetricsCollector.java
@@ -123,14 +123,12 @@ public void collectMetrics(long startTime) {
                     "",
                     System.currentTimeMillis() - mCurrT);
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.FAULT_DETECTION_COLLECTOR_ERROR,
-                    "",
-                    System.currentTimeMillis() - mCurrT);
             LOG.debug(
                     "Exception in Collecting FaultDetection Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.FAULT_DETECTION_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/NodeDetailsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/NodeDetailsCollector.java
@@ -14,13 +14,16 @@
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.performanceanalyzer.OpenSearchResources;
+import org.opensearch.performanceanalyzer.PerformanceAnalyzerApp;
 import org.opensearch.performanceanalyzer.config.overrides.ConfigOverridesHelper;
 import org.opensearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper;
 import org.opensearch.performanceanalyzer.metrics.AllMetrics.NodeDetailColumns;
 import org.opensearch.performanceanalyzer.metrics.AllMetrics.NodeRole;
 import org.opensearch.performanceanalyzer.metrics.MetricsConfiguration;
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 
 public class NodeDetailsCollector extends PerformanceAnalyzerMetricsCollector
         implements MetricsProcessor {
@@ -43,6 +46,8 @@ public void collectMetrics(long startTime) {
             return;
         }
 
+        long mCurrT = System.currentTimeMillis();
+
         StringBuilder value = new StringBuilder();
         value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds())
                 .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor);
@@ -62,6 +67,8 @@ public void collectMetrics(long startTime) {
             }
         } catch (IOException ioe) {
             LOG.error("Unable to serialize rca config overrides.", ioe);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CONFIG_OVERRIDES_SER_FAILED, "", 1);
         }
         value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor);
 
@@ -87,6 +94,10 @@ public void collectMetrics(long startTime) {
                     discoveryNodeIterator.next(), value, localNodeID, clusterManagerNode);
         }
         saveMetricValues(value.toString(), startTime);
+        PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
+                WriterMetrics.NODE_DETAILS_COLLECTOR_EXECUTION_TIME,
+                "",
+                System.currentTimeMillis() - mCurrT);
     }
 
     private void addMetricsToStringBuilder(

diff --git a/...ava/org/opensearch/performanceanalyzer/collectors/NodeStatsAllShardsMetricsCollector.java b/...ava/org/opensearch/performanceanalyzer/collectors/NodeStatsAllShardsMetricsCollector.java
@@ -29,6 +29,7 @@
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 import org.opensearch.performanceanalyzer.util.Utils;
 
 /**
@@ -146,6 +147,8 @@ public void collectMetrics(long startTime) {
             return;
         }
 
+        long mCurrT = System.currentTimeMillis();
+
         try {
             populateCurrentShards();
             populatePerShardStats(indicesService);
@@ -173,6 +176,11 @@ public void collectMetrics(long startTime) {
                         new NodeStatsMetricsAllShardsPerCollectionStatus(currentShardStats);
                 populateDiffMetricValue(
                         prevValue, currValue, startTime, shardId.getIndexName(), shardId.id());
+
+                PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
+                        WriterMetrics.NODE_STATS_ALL_SHARDS_METRICS_COLLECTOR_EXECUTION_TIME,
+                        "",
+                        System.currentTimeMillis() - mCurrT);
             }
         } catch (Exception ex) {
             LOG.debug(

diff --git a/...a/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java b/...a/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java
@@ -29,13 +29,14 @@
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 import org.opensearch.performanceanalyzer.util.Utils;
 
 /**
  * This collector collects metrics for fixed number of shards on a node in a single run. These
- * metrics are heavy weight metrics which have performance impacts on the performance of the node.
- * The number of shards is set via a cluster settings api. The parameter to set is
- * shardsPerCollection. The metrics will be populated for these many shards in a single run.
+ * metrics are heavy which have performance impacts on the performance of the node. The number of
+ * shards is set via a cluster settings api. The parameter to set is shardsPerCollection. The
+ * metrics will be populated for these many shards in a single run.
  */
 @SuppressWarnings("unchecked")
 public class NodeStatsFixedShardsMetricsCollector extends PerformanceAnalyzerMetricsCollector
@@ -166,6 +167,8 @@ public void collectMetrics(long startTime) {
             return;
         }
 
+        long mCurrT = System.currentTimeMillis();
+
         try {
             // reach the end of current shardId list. retrieve new shard list from IndexService
             if (!currentShardsIter.hasNext()) {
@@ -203,6 +206,11 @@ public void collectMetrics(long startTime) {
                             startTime,
                             currentIndexShardStats.getShardId().getIndexName(),
                             String.valueOf(currentIndexShardStats.getShardId().id()));
+
+                    PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
+                            WriterMetrics.NODE_STATS_FIXED_SHARDS_METRICS_COLLECTOR_EXECUTION_TIME,
+                            "",
+                            System.currentTimeMillis() - mCurrT);
                 }
             }
         } catch (Exception ex) {