Add collector time period to rca.conf

pa_config/rca.conf

@@ -58,8 +58,12 @@
       "io-total-throughput-in-bytes" : 250000.0,
       "io-total-syscallrate-per-second" : 0.1
     },
-    "cache-config": {
-      "field-data-cache-size-threshold" : 0.8,
+    //field data cache rca
+    "field-data-cache-rca": {
+      "field-data-cache-size-threshold" : 0.8
+    },
+    //shard request cache rca
+    "shard-request-cache-rca": {
       "shard-request-cache-threshold" : 0.9
     }
   },

pa_config/rca_idle_master.conf

@@ -69,8 +69,12 @@
       "io-total-throughput-cluster-percentage" : 0.3,
       "io-total-syscallrate-cluster-percentage" : 0.3
     },
-    "cache-config": {
-      "field-data-cache-size-threshold" : 0.8,
+    //field data cache rca
+    "field-data-cache-rca": {
+      "field-data-cache-size-threshold" : 0.8
+    },
+    //shard request cache rca
+    "shard-request-cache-rca": {
       "shard-request-cache-threshold" : 0.9
     }
   },

pa_config/rca_master.conf

@@ -69,8 +69,12 @@
       "io-total-throughput-cluster-percentage" : 0.3,
       "io-total-syscallrate-cluster-percentage" : 0.3
     },
-    "cache-config": {
-      "field-data-cache-size-threshold" : 0.8,
+    //field data cache rca
+    "field-data-cache-rca": {
+      "field-data-cache-size-threshold" : 0.8
+    },
+    //shard request cache rca
+    "shard-request-cache-rca": {
       "shard-request-cache-threshold" : 0.9
     }
   },

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/FieldDataCacheRcaConfig.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
+
+/**
+ * config object to store rca config settings for FieldDataCacheRca
+ */
+public class FieldDataCacheRcaConfig {
+    public static final String CONFIG_NAME = "field-data-cache-rca-config";
+
+    private Double fieldDataCacheSizeThreshold;
+    private Integer fieldDataCollectorTimePeriodInSec;
+
+    // Field data cache size threshold is 80%
+    public static final double DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD = 0.8;
+    // Metrics like eviction, hits are collected every 300 sec in field data cache rca
+    public static final int DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC = 300;
+
+    public FieldDataCacheRcaConfig(final RcaConf rcaConf) {
+        fieldDataCacheSizeThreshold =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.FIELD_DATA_CACHE_SIZE_THRESHOLD,
+                        DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD,
+                        (s) -> (s > 0),
+                        Double.class);
+        fieldDataCollectorTimePeriodInSec =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        (s) -> (s > 0),
+                        Integer.class);
+        if (fieldDataCacheSizeThreshold == null) {
+            fieldDataCacheSizeThreshold = DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD;
+        }
+        if (fieldDataCollectorTimePeriodInSec == null) {
+            fieldDataCollectorTimePeriodInSec = DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC;
+        }
+    }
+
+    public double getFieldDataCacheSizeThreshold() {
+        return fieldDataCacheSizeThreshold;
+    }
+
+    public int getFieldDataCollectorTimePeriodInSec() {
+        return fieldDataCollectorTimePeriodInSec;
+    }
+
+    public static class RCA_CONF_KEY_CONSTANTS {
+        public static final String FIELD_DATA_CACHE_SIZE_THRESHOLD = "field-data-cache-size-threshold";
+        public static final String FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC = "field-data-collector-time-period-in-sec";
+    }
+}

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageOldGenRcaConfig.java

@@ -30,7 +30,7 @@ public class HighHeapUsageOldGenRcaConfig {
   public static final String CONFIG_NAME = "high-heap-usage-old-gen-rca";
 
   public HighHeapUsageOldGenRcaConfig(final RcaConf rcaConf) {
-    topK = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.TOP_K, Integer.class);
+    topK = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.TOP_K, DEFAULT_TOP_K, (s) -> (s > 0), Integer.class);
     if (topK == null) {
       topK = DEFAULT_TOP_K;
     }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageYoungGenRcaConfig.java

@@ -30,8 +30,11 @@ public class HighHeapUsageYoungGenRcaConfig {
   public static final int DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC = 400;
 
   public HighHeapUsageYoungGenRcaConfig(final RcaConf rcaConf) {
-    promotionRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, Integer.class);
-    youngGenGcTimeThreshold = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.YOUNG_GEN_GC_TIME_THRES, Integer.class);
+    promotionRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
+            RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC, (s) -> (s > 0), Integer.class);
+    youngGenGcTimeThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
+            RCA_CONF_KEY_CONSTANTS.YOUNG_GEN_GC_TIME_THRES, DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC,
+            (s) -> (s > 0), Integer.class);
     if (promotionRateThreshold == null) {
       promotionRateThreshold = DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC;
     }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotNodeClusterRcaConfig.java

@@ -32,9 +32,9 @@ public class HotNodeClusterRcaConfig {
 
   public HotNodeClusterRcaConfig(final RcaConf rcaConf) {
     unbalancedResourceThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-        RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, Double.class);
+        RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, DEFAULT_UNBALANCED_RESOURCE_THRES, (s) -> (s > 0), Double.class);
     resourceUsageLowerBoundThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-        RCA_CONF_KEY_CONSTANTS.RESOURCE_USAGE_LOWER_BOUND_THRES, Double.class);
+        RCA_CONF_KEY_CONSTANTS.RESOURCE_USAGE_LOWER_BOUND_THRES, DEFAULT_RESOURCE_USAGE_LOWER_BOUND_THRES, (s) -> (s > 0), Double.class);
     if (unbalancedResourceThreshold == null) {
       unbalancedResourceThreshold = DEFAULT_UNBALANCED_RESOURCE_THRES;
     }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardClusterRcaConfig.java

@@ -15,11 +15,14 @@ public class HotShardClusterRcaConfig {
 
     public HotShardClusterRcaConfig(final RcaConf rcaConf) {
         cpuUtilizationClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_CLUSTER_THRESHOLD, Double.class);
+                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_CLUSTER_THRESHOLD,
+                DEFAULT_CPU_UTILIZATION_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class);
         ioTotThroughputClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_THROUGHPUT_CLUSTER_THRESHOLD, Double.class);
+                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_THROUGHPUT_CLUSTER_THRESHOLD,
+                DEFAULT_IO_TOTAL_THROUGHPUT_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class);
         ioTotSysCallRateClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_SYSCALLRATE_CLUSTER_THRESHOLD, Double.class);
+                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_SYSCALLRATE_CLUSTER_THRESHOLD,
+                DEFAULT_IO_TOTAL_SYSCALL_RATE_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class);
         if (cpuUtilizationClusterThreshold == null) {
             cpuUtilizationClusterThreshold = DEFAULT_CPU_UTILIZATION_CLUSTER_THRESHOLD;
         }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardRcaConfig.java

@@ -15,11 +15,14 @@ public class HotShardRcaConfig {
 
     public HotShardRcaConfig(final RcaConf rcaConf) {
         cpuUtilizationThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_THRESHOLD, Double.class);
+                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_THRESHOLD,
+                DEFAULT_CPU_UTILIZATION_THRESHOLD, (s) -> (s > 0), Double.class);
         ioTotThroughputThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_THROUGHPUT_THRESHOLD_IN_BYTES, Double.class);
+                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_THROUGHPUT_THRESHOLD_IN_BYTES,
+                DEFAULT_IO_TOTAL_THROUGHPUT_THRESHOLD_IN_BYTE_PER_SEC, (s) -> (s > 0), Double.class);
         ioTotSysCallRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_SYSCALL_RATE_THRESHOLD_PER_SECOND, Double.class);
+                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_SYSCALL_RATE_THRESHOLD_PER_SECOND,
+                DEFAULT_IO_TOTAL_SYSCALL_RATE_THRESHOLD_PER_SEC, (s) -> (s > 0), Double.class);
         if (cpuUtilizationThreshold == null) {
             cpuUtilizationThreshold = DEFAULT_CPU_UTILIZATION_THRESHOLD;
         }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java

@@ -24,7 +24,8 @@ public class QueueRejectionRcaConfig {
 
   public QueueRejectionRcaConfig(final RcaConf rcaConf) {
     rejectionTimePeriodInSeconds = rcaConf.readRcaConfig(
-        CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.REJECTION_TIME_PERIOD_IN_SECONDS, Integer.class);
+        CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.REJECTION_TIME_PERIOD_IN_SECONDS,
+        DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS, (s) -> (s > 0), Integer.class);
     if (rejectionTimePeriodInSeconds == null) {
       rejectionTimePeriodInSeconds = DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS;
     }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/ShardRequestCacheRcaConfig.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
+
+/**
+ * config object to store rca config settings for ShardRequestCacheRca
+ */
+public class ShardRequestCacheRcaConfig {
+    public static final String CONFIG_NAME = "shard-request-cache-rca-config";
+
+    private Double shardRequestCacheSizeThreshold;
+    private Integer shardRequestCollectorTimePeriodInSec;
+
+    // Shard request cache size threshold is 90%
+    public static final double DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD = 0.9;
+    // Metrics like eviction, hits are collected every 300 sec in shard request cache rca
+    public static final int DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC = 300;
+
+    public ShardRequestCacheRcaConfig(final RcaConf rcaConf) {
+        shardRequestCacheSizeThreshold =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_CACHE_SIZE_THRESHOLD,
+                        DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD,
+                        (s) -> (s > 0),
+                        Double.class);
+        shardRequestCollectorTimePeriodInSec =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        (s) -> (s > 0),
+                        Integer.class);
+        if (shardRequestCacheSizeThreshold == null) {
+            shardRequestCacheSizeThreshold = DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD;
+        }
+        if (shardRequestCollectorTimePeriodInSec == null) {
+            shardRequestCollectorTimePeriodInSec = DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC;
+        }
+    }
+
+    public double getShardRequestCacheSizeThreshold() {
+        return shardRequestCacheSizeThreshold;
+    }
+
+    public int getShardRequestCollectorTimePeriodInSec() {
+        return shardRequestCollectorTimePeriodInSec;
+    }
+
+    public static class RCA_CONF_KEY_CONSTANTS {
+        public static final String SHARD_REQUEST_CACHE_SIZE_THRESHOLD = "shard-request-cache-threshold";
+        public static final String SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC = "shard-request-collector-time-period-in-sec";
+    }
+}

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java

@@ -18,14 +18,15 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.RcaControllerHelper;
-import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.CacheConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.DeciderConfig;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.FieldDataCacheRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageOldGenRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageYoungGenRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotNodeClusterRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardClusterRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.QueueRejectionRcaConfig;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.ShardRequestCacheRcaConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.RcaConsts;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
@@ -49,6 +50,8 @@
 import java.util.Map;
 import java.util.Scanner;
 import java.util.Set;
+import java.util.function.Predicate;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -162,8 +165,12 @@ public HotShardClusterRcaConfig getHotShardClusterRcaConfig() {
     return new HotShardClusterRcaConfig(this);
   }
 
-  public CacheConfig getCacheConfig() {
-    return new CacheConfig(this);
+  public FieldDataCacheRcaConfig getFieldDataCacheRcaConfig() {
+    return new FieldDataCacheRcaConfig(this);
+  }
+
+  public ShardRequestCacheRcaConfig getShardRequestCacheRcaConfig() {
+    return new ShardRequestCacheRcaConfig(this);
   }
 
   public DeciderConfig getDeciderConfig() {
@@ -186,21 +193,29 @@ public Map<String, Object> getRcaConfigSettings() {
     return ImmutableMap.copyOf(conf.getRcaConfigSettings());
   }
 
+  public <T> T readRcaConfig(String rcaName, String key, T defaultValue, Class<? extends T> clazz) {
+    return readRcaConfig(rcaName, key, defaultValue, (s) -> true, clazz);
+  }
+
   @SuppressWarnings("unchecked")
-  public <T> T readRcaConfig(String rcaName, String key, Class<? extends T> clazz) {
+  public <T> T readRcaConfig(String rcaName, String key, T defaultValue, Predicate<T> validator, Class<? extends T> clazz) {
     T setting = null;
     try {
       Map<String, Object> rcaObj = null;
       if (conf.getRcaConfigSettings() != null
-          && conf.getRcaConfigSettings().containsKey(rcaName)
-          && conf.getRcaConfigSettings().get(rcaName) != null) {
+              && conf.getRcaConfigSettings().containsKey(rcaName)
+              && conf.getRcaConfigSettings().get(rcaName) != null) {
         rcaObj = (Map<String, Object>) conf.getRcaConfigSettings().get(rcaName);
       }
 
       if (rcaObj != null
-          && rcaObj.containsKey(key)
-          && rcaObj.get(key) != null) {
+              && rcaObj.containsKey(key)
+              && rcaObj.get(key) != null) {
         setting = clazz.cast(rcaObj.get(key));
+        if (!validator.test(setting)) {
+          LOG.error("Config value: [{}] provided for key: [{}] is invalid", setting, key);
+          return defaultValue;
+        }
       }
     } catch (ClassCastException ne) {
       LOG.error("rca.conf contains value in invalid format, trace : {}", ne.getMessage());