Add IT for cache tuning (#382)

* Add IT for cache tuning

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/metrics/Cache_FieldData_Eviction.java

@@ -19,7 +19,9 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric;
 
 public class Cache_FieldData_Eviction extends Metric {
+  public static final String NAME = AllMetrics.ShardStatsValue.CACHE_FIELDDATA_EVICTION.toString();
+
   public Cache_FieldData_Eviction(long evaluationIntervalSeconds) {
-    super(AllMetrics.ShardStatsValue.CACHE_FIELDDATA_EVICTION.toString(), evaluationIntervalSeconds);
+    super(NAME, evaluationIntervalSeconds);
   }
 }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/metrics/Cache_Request_Eviction.java

@@ -19,7 +19,9 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric;
 
 public class Cache_Request_Eviction extends Metric {
+  public static final String NAME = AllMetrics.ShardStatsValue.CACHE_REQUEST_EVICTION.toString();
+
   public Cache_Request_Eviction(long evaluationIntervalSeconds) {
-    super(AllMetrics.ShardStatsValue.CACHE_REQUEST_EVICTION.toString(), evaluationIntervalSeconds);
+    super(NAME, evaluationIntervalSeconds);
   }
 }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/metrics/Cache_Request_Hit.java

@@ -19,7 +19,9 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric;
 
 public class Cache_Request_Hit extends Metric {
+  public static final String NAME = AllMetrics.ShardStatsValue.CACHE_REQUEST_HIT.toString();
+
   public Cache_Request_Hit(long evaluationIntervalSeconds) {
-    super(AllMetrics.ShardStatsValue.CACHE_REQUEST_HIT.toString(), evaluationIntervalSeconds);
+    super(NAME, evaluationIntervalSeconds);
   }
 }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/metrics/Cache_Request_Miss.java

@@ -19,7 +19,9 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric;
 
 public class Cache_Request_Miss extends Metric {
+  public static final String NAME = AllMetrics.ShardStatsValue.CACHE_REQUEST_MISS.toString();
+
   public Cache_Request_Miss(long evaluationIntervalSeconds) {
-    super(AllMetrics.ShardStatsValue.CACHE_REQUEST_MISS.toString(), evaluationIntervalSeconds);
+    super(NAME, evaluationIntervalSeconds);
   }
 }

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/RcaItCacheTuning.java

@@ -0,0 +1,241 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ *  A copy of the License is located at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  or in the "license" file accompanying this file. This file is distributed
+ *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *  express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning;
+
+import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning.RcaItCacheTuning.INDEX_NAME;
+import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning.RcaItCacheTuning.SHARD_ID;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_FieldData_Eviction;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_FieldData_Size;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Max_Size;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Request_Eviction;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Request_Hit;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Request_Size;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.RcaItMarker;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AClusterType;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AErrorPatternIgnored;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AExpect;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AMetric;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ARcaGraph;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATable;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATuple;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.ClusterType;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.Consts;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.HostTag;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.runners.RcaItNotEncryptedRunner;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning.validator.FieldDataCacheValidator;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning.validator.ShardRequestCacheValidator;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.ElasticSearchAnalysisGraph;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.FieldDataCacheClusterRca;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.ShardRequestCacheClusterRca;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+@Category(RcaItMarker.class)
+@RunWith(RcaItNotEncryptedRunner.class)
+@AClusterType(ClusterType.MULTI_NODE_CO_LOCATED_MASTER)
+@ARcaGraph(ElasticSearchAnalysisGraph.class)
+@AMetric(
+    name = Cache_FieldData_Size.class,
+    dimensionNames = {
+      AllMetrics.CommonDimension.Constants.INDEX_NAME_VALUE,
+      AllMetrics.CommonDimension.Constants.SHARDID_VALUE
+    },
+    tables = {
+      @ATable(
+          hostTag = HostTag.DATA_0,
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 8500.0, avg = 8500.0, min = 8500.0, max = 8500.0),
+          }),
+      @ATable(
+          hostTag = {HostTag.ELECTED_MASTER},
+          tuple = {
+            @ATuple(
+                dimensionValues = {"Index1", "1"},
+                sum = 100.0, avg = 100.0, min = 100.0, max = 100.0),
+          })
+    })
+@AMetric(
+    name = Cache_FieldData_Eviction.class,
+    dimensionNames = {
+      AllMetrics.CommonDimension.Constants.INDEX_NAME_VALUE,
+      AllMetrics.CommonDimension.Constants.SHARDID_VALUE
+    },
+    tables = {
+      @ATable(
+          hostTag = HostTag.DATA_0,
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 1.0, avg = 1.0, min = 1.0, max = 1.0),
+          }),
+      @ATable(
+          hostTag = {HostTag.ELECTED_MASTER},
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 0.0, avg = 0.0, min = 0.0, max = 0.0),
+          })
+    })
+@AMetric(
+    name = Cache_Request_Size.class,
+    dimensionNames = {
+      AllMetrics.CommonDimension.Constants.INDEX_NAME_VALUE,
+      AllMetrics.CommonDimension.Constants.SHARDID_VALUE
+    },
+    tables = {
+      @ATable(
+          hostTag = HostTag.DATA_0,
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 100.0, avg = 100.0, min = 100.0, max = 100.0),
+          }),
+      @ATable(
+          hostTag = {HostTag.ELECTED_MASTER},
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 50.0, avg = 50.0, min = 50.0, max = 50.0),
+          })
+    })
+@AMetric(
+    name = Cache_Request_Eviction.class,
+    dimensionNames = {
+      AllMetrics.CommonDimension.Constants.INDEX_NAME_VALUE,
+      AllMetrics.CommonDimension.Constants.SHARDID_VALUE
+    },
+    tables = {
+      @ATable(
+          hostTag = HostTag.DATA_0,
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 1.0, avg = 1.0, min = 1.0, max = 1.0),
+          }),
+      @ATable(
+          hostTag = {HostTag.ELECTED_MASTER},
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 0.0, avg = 0.0, min = 0.0, max = 0.0),
+          })
+    })
+@AMetric(
+    name = Cache_Request_Hit.class,
+    dimensionNames = {
+      AllMetrics.CommonDimension.Constants.INDEX_NAME_VALUE,
+      AllMetrics.CommonDimension.Constants.SHARDID_VALUE
+    },
+    tables = {
+      @ATable(
+          hostTag = HostTag.DATA_0,
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 1.0, avg = 1.0, min = 1.0, max = 1.0),
+          }),
+      @ATable(
+          hostTag = {HostTag.ELECTED_MASTER},
+          tuple = {
+            @ATuple(
+                dimensionValues = {INDEX_NAME, SHARD_ID},
+                sum = 0.0, avg = 0.0, min = 0.0, max = 0.0),
+          })
+    })
+@AMetric(
+    name = Cache_Max_Size.class,
+    dimensionNames = {AllMetrics.CacheConfigDimension.Constants.TYPE_VALUE},
+    tables = {
+      @ATable(
+          hostTag = HostTag.DATA_0,
+          tuple = {
+            @ATuple(
+                dimensionValues = {AllMetrics.CacheType.Constants.FIELD_DATA_CACHE_NAME},
+                sum = 10000.0, avg = 10000.0, min = 10000.0, max = 10000.0),
+            @ATuple(
+                dimensionValues = {AllMetrics.CacheType.Constants.SHARD_REQUEST_CACHE_NAME},
+                sum = 100.0, avg = 100.0, min = 100.0, max = 100.0),
+            @ATuple(
+                dimensionValues = {AllMetrics.CacheType.Constants.SHARD_REQUEST_CACHE_NAME},
+                sum = 100.0, avg = 100.0, min = 100.0, max = 100.0)
+          }),
+      @ATable(
+          hostTag = {HostTag.ELECTED_MASTER},
+          tuple = {
+            @ATuple(
+                dimensionValues = {AllMetrics.CacheType.Constants.FIELD_DATA_CACHE_NAME},
+                sum = 10000.0, avg = 10000.0, min = 10000.0, max = 10000.0),
+            @ATuple(
+                dimensionValues = {AllMetrics.CacheType.Constants.SHARD_REQUEST_CACHE_NAME},
+                sum = 100.0, avg = 100.0, min = 100.0, max = 100.0)
+          })
+    })
+public class RcaItCacheTuning {
+  public static final String INDEX_NAME = "MockIndex";
+  public static final String SHARD_ID = "1";
+
+  // Test FieldDataCacheClusterRca.
+  // This rca should be un-healthy when cache size is higher than threshold with evictions.
+  // TODO : extend this integ test to cover Decision Maker framework and queue remediation actions
+  @Test
+  @AExpect(
+      what = AExpect.Type.REST_API,
+      on = HostTag.ELECTED_MASTER,
+      validator = FieldDataCacheValidator.class,
+      forRca = FieldDataCacheClusterRca.class,
+      timeoutSeconds = 700)
+  @AErrorPatternIgnored(
+      pattern = "AggregateMetric:gather()",
+      reason = "CPU metrics are expected to be missing in this integ test")
+  @AErrorPatternIgnored(
+      pattern = "Metric:gather()",
+      reason = "Metrics are expected to be missing in this integ test")
+  @AErrorPatternIgnored(
+          pattern = "NodeConfigCacheReaderUtil",
+          reason = "Node Config Cache are expected to be missing in this integ test.")
+  @AErrorPatternIgnored(
+          pattern = "CacheUtil:getCacheMaxSize()",
+          reason = "Node Config Cache are are expected to be missing during startup.")
+  public void testFieldDataCacheRca() {}
+
+  // Test ShardRequestCacheClusterRca.
+  // This rca should be un-healthy when cache size is higher than threshold with evictions and hits.
+  @Test
+  @AExpect(
+      what = AExpect.Type.REST_API,
+      on = HostTag.ELECTED_MASTER,
+      validator = ShardRequestCacheValidator.class,
+      forRca = ShardRequestCacheClusterRca.class,
+      timeoutSeconds = 700)
+  @AErrorPatternIgnored(
+          pattern = "AggregateMetric:gather()",
+          reason = "CPU metrics are expected to be missing in this integ test")
+  @AErrorPatternIgnored(
+          pattern = "Metric:gather()",
+          reason = "Metrics are expected to be missing in this integ test")
+  @AErrorPatternIgnored(
+          pattern = "NodeConfigCacheReaderUtil",
+          reason = "Node config cache metrics are expected to be missing in this integ test.")
+  @AErrorPatternIgnored(
+          pattern = "CacheUtil:getCacheMaxSize()",
+          reason = "Node Config Cache are are expected to be missing during startup.")
+  public void testShardRequestCacheRca() {}
+}

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/validator/FieldDataCacheValidator.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ *  A copy of the License is located at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  or in the "license" file accompanying this file. This file is distributed
+ *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *  express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning.validator;
+
+import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotNodeSummary.SQL_SCHEMA_CONSTANTS.HOST_IP_ADDRESS_COL_NAME;
+import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotNodeSummary.SQL_SCHEMA_CONSTANTS.NODE_ID_COL_NAME;
+import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotResourceSummary.SQL_SCHEMA_CONSTANTS.RESOURCE_TYPE_COL_NAME;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotClusterSummary;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotNodeSummary;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotResourceSummary;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api.IValidator;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.util.JsonParserUtil;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache.FieldDataCacheRca;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.FieldDataCacheClusterRca;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+
+public class FieldDataCacheValidator implements IValidator {
+    long startTime;
+
+    public FieldDataCacheValidator() {
+        startTime = System.currentTimeMillis();
+    }
+
+    /**
+     * {"rca_name":"FieldDataCacheClusterRca",
+     * "timestamp":1596557050522,
+     * "state":"unhealthy",
+     * "HotClusterSummary":[
+     * {"number_of_nodes":1,"number_of_unhealthy_nodes":1}
+     * ]}
+     */
+    @Override
+    public boolean check(JsonElement response) {
+        JsonArray array = response.getAsJsonObject().get("data").getAsJsonArray();
+        if (array.size() == 0) {
+            return false;
+        }
+        for (int i = 0; i < array.size(); i++) {
+            JsonObject object = array.get(i).getAsJsonObject();
+            if (object.get("rca_name").getAsString().equals(FieldDataCacheClusterRca.RCA_TABLE_NAME)) {
+                return checkClusterRca(object);
+            }
+        }
+        return false;
+    }
+
+    /**
+     * {"rca_name":"FieldDataCacheClusterRca",
+     *  "timestamp":1596557050522,
+     *  "state":"unhealthy",
+     *  "HotClusterSummary":[{"number_of_nodes":1,"number_of_unhealthy_nodes":1}]
+     * }
+     */
+    boolean checkClusterRca(final JsonObject rcaObject) {
+        if (!"unhealthy".equals(rcaObject.get("state").getAsString())) {
+            return false;
+        }
+        Assert.assertEquals(1,
+                JsonParserUtil.getSummaryJsonSize(rcaObject, HotClusterSummary.HOT_CLUSTER_SUMMARY_TABLE));
+        JsonObject clusterSummaryJson =
+                JsonParserUtil.getSummaryJson(rcaObject, HotClusterSummary.HOT_CLUSTER_SUMMARY_TABLE, 0);
+        Assert.assertNotNull(clusterSummaryJson);
+        Assert.assertEquals(1, clusterSummaryJson.get("number_of_unhealthy_nodes").getAsInt());
+
+        Assert.assertEquals(1,
+                JsonParserUtil.getSummaryJsonSize(clusterSummaryJson, HotNodeSummary.HOT_NODE_SUMMARY_TABLE));
+        JsonObject nodeSummaryJson =
+                JsonParserUtil.getSummaryJson(clusterSummaryJson, HotNodeSummary.HOT_NODE_SUMMARY_TABLE, 0);
+        Assert.assertNotNull(nodeSummaryJson);
+        Assert.assertEquals("DATA_0", nodeSummaryJson.get(NODE_ID_COL_NAME).getAsString());
+        Assert.assertEquals("127.0.0.1", nodeSummaryJson.get(HOST_IP_ADDRESS_COL_NAME).getAsString());
+
+        Assert.assertEquals(1,
+                JsonParserUtil.getSummaryJsonSize(nodeSummaryJson, HotResourceSummary.HOT_RESOURCE_SUMMARY_TABLE));
+        JsonObject resourceSummaryJson =
+                JsonParserUtil.getSummaryJson(nodeSummaryJson, HotResourceSummary.HOT_RESOURCE_SUMMARY_TABLE, 0);
+        Assert.assertNotNull(resourceSummaryJson);
+        Assert.assertEquals("field data cache", resourceSummaryJson.get(RESOURCE_TYPE_COL_NAME).getAsString());
+        return true;
+    }
+}