FieldData and Shard Request Cache RCA

pa_config/rca.conf

@@ -54,6 +54,10 @@
       "cpu-utilization" : 0.01,
       "io-total-throughput-in-bytes" : 250000.0,
       "io-total-syscallrate-per-second" : 0.1
+    },
+    "cache-config": {
+      "field-data-cache-size-threshold" : 0.8,
+      "shard-request-cache-threshold" : 0.9
     }
   },
 

pa_config/rca_idle_master.conf

@@ -65,6 +65,10 @@
       "cpu-utilization-cluster-percentage" : 0.3,
       "io-total-throughput-cluster-percentage" : 0.3,
       "io-total-syscallrate-cluster-percentage" : 0.3
+    },
+    "cache-config": {
+      "field-data-cache-size-threshold" : 0.8,
+      "shard-request-cache-threshold" : 0.9
     }
   },
 

pa_config/rca_master.conf

@@ -65,6 +65,10 @@
       "cpu-utilization-cluster-percentage" : 0.3,
       "io-total-throughput-cluster-percentage" : 0.3,
       "io-total-syscallrate-cluster-percentage" : 0.3
+    },
+    "cache-config": {
+      "field-data-cache-size-threshold" : 0.8,
+      "shard-request-cache-threshold" : 0.9
     }
   },
 

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/CacheConfig.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
+
+public class CacheConfig {
+    public static final String CONFIG_NAME = "cache-config";
+
+    private Double fieldDataCacheSizeThreshold;
+    private Double shardRequestCacheSizeThreshold;
+
+    public static final double DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD = 0.8;
+    public static final double DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD = 0.9;
+
+    public CacheConfig(final RcaConf rcaConf) {
+        fieldDataCacheSizeThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
+                RCA_CONF_KEY_CONSTANTS.FIELD_DATA_CACHE_SIZE_THRESHOLD, Double.class);
+        shardRequestCacheSizeThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
+                RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_CACHE_SIZE_THRESHOLD, Double.class);
+        if (fieldDataCacheSizeThreshold == null) {
+            fieldDataCacheSizeThreshold = DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD;
+        }
+        if (shardRequestCacheSizeThreshold == null) {
+            shardRequestCacheSizeThreshold = DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD;
+        }
+    }
+
+    public double getFieldDataCacheSizeThreshold() {
+        return fieldDataCacheSizeThreshold;
+    }
+
+    public double getShardRequestCacheSizeThreshold() {
+        return shardRequestCacheSizeThreshold;
+    }
+
+    public static class RCA_CONF_KEY_CONSTANTS {
+        public static final String FIELD_DATA_CACHE_SIZE_THRESHOLD = "field-data-cache-size-threshold";
+        public static final String SHARD_REQUEST_CACHE_SIZE_THRESHOLD = "shard-request-cache-threshold";
+    }
+}

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/summaries/ResourceUtil.java

@@ -59,6 +59,17 @@ public class ResourceUtil {
       .setResourceEnum(ResourceEnum.SEARCH_THREADPOOL)
       .setMetricEnum(MetricEnum.QUEUE_REJECTION).build();
 
+  // cache
+  public static final Resource FIELD_DATA_CACHE_EVICTION = Resource.newBuilder()
+          .setResourceEnum(ResourceEnum.FIELD_DATA_CACHE)
+          .setMetricEnum(MetricEnum.CACHE_EVICTION).build();
+  public static final Resource SHARD_REQUEST_CACHE_EVICTION = Resource.newBuilder()
+          .setResourceEnum(ResourceEnum.SHARD_REQUEST_CACHE)
+          .setMetricEnum(MetricEnum.CACHE_EVICTION).build();
+  public static final Resource SHARD_REQUEST_CACHE_HIT = Resource.newBuilder()
+          .setResourceEnum(ResourceEnum.SHARD_REQUEST_CACHE)
+          .setMetricEnum(MetricEnum.CACHE_HIT).build();
+
   /**
    * Read the resourceType name from the ResourceType object
    * @param resource grpc Resource object

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java

@@ -16,6 +16,7 @@
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core;
 
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.CacheConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageOldGenRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageYoungGenRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotNodeClusterRcaConfig;
@@ -131,6 +132,10 @@ public HotShardClusterRcaConfig getHotShardClusterRcaConfig() {
     return new HotShardClusterRcaConfig(this);
   }
 
+  public CacheConfig getCacheConfig() {
+    return new CacheConfig(this);
+  }
+
   public List<String> getMutedRcaList() {
     return conf.getMutedRcaList();
   }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/CacheUtil.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.flow_units.MetricFlowUnit;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class CacheUtil {
+    private static final Logger LOG = LogManager.getLogger(CacheUtil.class);
+
+    public static Double getTotalSizeInKB(final Metric cacheSizeGroupByOperation) {
+        double sizeTotalInKB = 0;
+
+        if (cacheSizeGroupByOperation.getFlowUnits().size() > 0) {
+            // we expect the Metric to have single flow unit since it is consumed locally
+            MetricFlowUnit flowUnit = cacheSizeGroupByOperation.getFlowUnits().get(0);
+            if (flowUnit.isEmpty() || flowUnit.getData() == null) {
+                return sizeTotalInKB;
+            }
+
+            // since the flow unit data is aggregated, we should have a single value
+            if (flowUnit.getData().size() > 0) {
+                double size = flowUnit.getData().get(0).getValue(MetricsDB.SUM, Double.class);
+                if (Double.isNaN(size)) {
+                    LOG.error("Failed to parse metric in FlowUnit from {}", cacheSizeGroupByOperation.getClass().getName());
+                } else {
+                    sizeTotalInKB += size / 1024.0;
+                }
+            }
+        }
+        return sizeTotalInKB;
+    }
+
+    public static Boolean isSizeThresholdExceeded(final Metric cacheSizeGroupByOperation,
+                                                  final Metric cacheMaxSizeGroupByOperation,
+                                                  double threshold_percentage) {
+        double cacheSize = getTotalSizeInKB(cacheSizeGroupByOperation);
+        double cacheMaxSize = getTotalSizeInKB(cacheMaxSizeGroupByOperation);
+        return cacheSize != 0 && cacheMaxSize != 0 && (cacheSize > cacheMaxSize * threshold_percentage);
+    }
+}
+