diff --git a/pa_config/rca.conf b/pa_config/rca.conf index 4899f740b..4d46e9478 100644 --- a/pa_config/rca.conf +++ b/pa_config/rca.conf @@ -58,8 +58,12 @@ "io-total-throughput-in-bytes" : 250000.0, "io-total-syscallrate-per-second" : 0.1 }, - "cache-config": { - "field-data-cache-size-threshold" : 0.8, + // field data cache rca + "field-data-cache-rca": { + "field-data-cache-size-threshold" : 0.8 + }, + // shard request cache rca + "shard-request-cache-rca": { "shard-request-cache-threshold" : 0.9 } }, diff --git a/pa_config/rca_idle_master.conf b/pa_config/rca_idle_master.conf index 33eac0c66..925115dbf 100644 --- a/pa_config/rca_idle_master.conf +++ b/pa_config/rca_idle_master.conf @@ -69,8 +69,12 @@ "io-total-throughput-cluster-percentage" : 0.3, "io-total-syscallrate-cluster-percentage" : 0.3 }, - "cache-config": { - "field-data-cache-size-threshold" : 0.8, + // field data cache rca + "field-data-cache-rca": { + "field-data-cache-size-threshold" : 0.8 + }, + // shard request cache rca + "shard-request-cache-rca": { "shard-request-cache-threshold" : 0.9 } }, diff --git a/pa_config/rca_master.conf b/pa_config/rca_master.conf index 4dbf0ae2c..791c1e3de 100644 --- a/pa_config/rca_master.conf +++ b/pa_config/rca_master.conf @@ -69,8 +69,12 @@ "io-total-throughput-cluster-percentage" : 0.3, "io-total-syscallrate-cluster-percentage" : 0.3 }, - "cache-config": { - "field-data-cache-size-threshold" : 0.8, + // field data cache rca + "field-data-cache-rca": { + "field-data-cache-size-threshold" : 0.8 + }, + // shard request cache rca + "shard-request-cache-rca": { "shard-request-cache-threshold" : 0.9 } }, diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/CacheConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/CacheConfig.java deleted file mode 100644 index 8b170a553..000000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/CacheConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; - -public class CacheConfig { - public static final String CONFIG_NAME = "cache-config"; - - private Double fieldDataCacheSizeThreshold; - private Double shardRequestCacheSizeThreshold; - - public static final double DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD = 0.8; - public static final double DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD = 0.9; - - public CacheConfig(final RcaConf rcaConf) { - fieldDataCacheSizeThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - RCA_CONF_KEY_CONSTANTS.FIELD_DATA_CACHE_SIZE_THRESHOLD, Double.class); - shardRequestCacheSizeThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_CACHE_SIZE_THRESHOLD, Double.class); - if (fieldDataCacheSizeThreshold == null) { - fieldDataCacheSizeThreshold = DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD; - } - if (shardRequestCacheSizeThreshold == null) { - shardRequestCacheSizeThreshold = DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD; - } - } - - public double getFieldDataCacheSizeThreshold() { - return fieldDataCacheSizeThreshold; - } - - public double getShardRequestCacheSizeThreshold() { - return shardRequestCacheSizeThreshold; - } - - public static class RCA_CONF_KEY_CONSTANTS { - public static final String FIELD_DATA_CACHE_SIZE_THRESHOLD = "field-data-cache-size-threshold"; - public static final String SHARD_REQUEST_CACHE_SIZE_THRESHOLD = "shard-request-cache-threshold"; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/FieldDataCacheRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/FieldDataCacheRcaConfig.java new file mode 100644 index 000000000..010d35101 --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/FieldDataCacheRcaConfig.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; + +/** + * config object to store rca config settings for FieldDataCacheRca + */ +public class FieldDataCacheRcaConfig { + public static final String CONFIG_NAME = "field-data-cache-rca-config"; + + private Double fieldDataCacheSizeThreshold; + private Integer fieldDataCollectorTimePeriodInSec; + + // Field data cache size threshold is 80% + public static final double DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD = 0.8; + // Metrics like eviction, hits are collected every 300 sec in field data cache rca + public static final int DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC = 300; + + public FieldDataCacheRcaConfig(final RcaConf rcaConf) { + fieldDataCacheSizeThreshold = + rcaConf.readRcaConfig( + CONFIG_NAME, + RCA_CONF_KEY_CONSTANTS.FIELD_DATA_CACHE_SIZE_THRESHOLD, + DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD, + (s) -> (s > 0), + Double.class); + fieldDataCollectorTimePeriodInSec = + rcaConf.readRcaConfig( + CONFIG_NAME, + RCA_CONF_KEY_CONSTANTS.FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC, + DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC, + (s) -> (s > 0), + Integer.class); + } + + public double getFieldDataCacheSizeThreshold() { + return fieldDataCacheSizeThreshold; + } + + public int getFieldDataCollectorTimePeriodInSec() { + return fieldDataCollectorTimePeriodInSec; + } + + public static class RCA_CONF_KEY_CONSTANTS { + public static final String FIELD_DATA_CACHE_SIZE_THRESHOLD = "field-data-cache-size-threshold"; + public static final String FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC = "field-data-collector-time-period-in-sec"; + } +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageOldGenRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageOldGenRcaConfig.java index 6c7e9697b..59f892397 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageOldGenRcaConfig.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageOldGenRcaConfig.java @@ -30,10 +30,7 @@ public class HighHeapUsageOldGenRcaConfig { public static final String CONFIG_NAME = "high-heap-usage-old-gen-rca"; public HighHeapUsageOldGenRcaConfig(final RcaConf rcaConf) { - topK = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.TOP_K, Integer.class); - if (topK == null) { - topK = DEFAULT_TOP_K; - } + topK = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.TOP_K, DEFAULT_TOP_K, (s) -> (s > 0), Integer.class); } public int getTopK() { diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageYoungGenRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageYoungGenRcaConfig.java index 298483c01..9e240fe10 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageYoungGenRcaConfig.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageYoungGenRcaConfig.java @@ -30,14 +30,11 @@ public class HighHeapUsageYoungGenRcaConfig { public static final int DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC = 400; public HighHeapUsageYoungGenRcaConfig(final RcaConf rcaConf) { - promotionRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, Integer.class); - youngGenGcTimeThreshold = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.YOUNG_GEN_GC_TIME_THRES, Integer.class); - if (promotionRateThreshold == null) { - promotionRateThreshold = DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC; - } - if (youngGenGcTimeThreshold == null) { - youngGenGcTimeThreshold = DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC; - } + promotionRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME, + RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC, (s) -> (s > 0), Integer.class); + youngGenGcTimeThreshold = rcaConf.readRcaConfig(CONFIG_NAME, + RCA_CONF_KEY_CONSTANTS.YOUNG_GEN_GC_TIME_THRES, DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC, + (s) -> (s > 0), Integer.class); } public int getPromotionRateThreshold() { diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotNodeClusterRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotNodeClusterRcaConfig.java index 94acb1196..4e4fcafe3 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotNodeClusterRcaConfig.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotNodeClusterRcaConfig.java @@ -32,15 +32,9 @@ public class HotNodeClusterRcaConfig { public HotNodeClusterRcaConfig(final RcaConf rcaConf) { unbalancedResourceThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, Double.class); + RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, DEFAULT_UNBALANCED_RESOURCE_THRES, (s) -> (s > 0), Double.class); resourceUsageLowerBoundThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - RCA_CONF_KEY_CONSTANTS.RESOURCE_USAGE_LOWER_BOUND_THRES, Double.class); - if (unbalancedResourceThreshold == null) { - unbalancedResourceThreshold = DEFAULT_UNBALANCED_RESOURCE_THRES; - } - if (resourceUsageLowerBoundThreshold == null) { - resourceUsageLowerBoundThreshold = DEFAULT_RESOURCE_USAGE_LOWER_BOUND_THRES; - } + RCA_CONF_KEY_CONSTANTS.RESOURCE_USAGE_LOWER_BOUND_THRES, DEFAULT_RESOURCE_USAGE_LOWER_BOUND_THRES, (s) -> (s > 0), Double.class); } public double getUnbalancedResourceThreshold() { diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardClusterRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardClusterRcaConfig.java index 8b6d7a6f4..bdacce1ad 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardClusterRcaConfig.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardClusterRcaConfig.java @@ -15,20 +15,14 @@ public class HotShardClusterRcaConfig { public HotShardClusterRcaConfig(final RcaConf rcaConf) { cpuUtilizationClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_CLUSTER_THRESHOLD, Double.class); + HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_CLUSTER_THRESHOLD, + DEFAULT_CPU_UTILIZATION_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class); ioTotThroughputClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_THROUGHPUT_CLUSTER_THRESHOLD, Double.class); + HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_THROUGHPUT_CLUSTER_THRESHOLD, + DEFAULT_IO_TOTAL_THROUGHPUT_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class); ioTotSysCallRateClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_SYSCALLRATE_CLUSTER_THRESHOLD, Double.class); - if (cpuUtilizationClusterThreshold == null) { - cpuUtilizationClusterThreshold = DEFAULT_CPU_UTILIZATION_CLUSTER_THRESHOLD; - } - if (ioTotThroughputClusterThreshold == null) { - ioTotThroughputClusterThreshold = DEFAULT_IO_TOTAL_THROUGHPUT_CLUSTER_THRESHOLD; - } - if (ioTotSysCallRateClusterThreshold == null) { - ioTotSysCallRateClusterThreshold = DEFAULT_IO_TOTAL_SYSCALL_RATE_CLUSTER_THRESHOLD; - } + HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_SYSCALLRATE_CLUSTER_THRESHOLD, + DEFAULT_IO_TOTAL_SYSCALL_RATE_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class); } public double getCpuUtilizationClusterThreshold() { diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardRcaConfig.java index 3da7cc424..012151773 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardRcaConfig.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardRcaConfig.java @@ -15,20 +15,14 @@ public class HotShardRcaConfig { public HotShardRcaConfig(final RcaConf rcaConf) { cpuUtilizationThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_THRESHOLD, Double.class); + HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_THRESHOLD, + DEFAULT_CPU_UTILIZATION_THRESHOLD, (s) -> (s > 0), Double.class); ioTotThroughputThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_THROUGHPUT_THRESHOLD_IN_BYTES, Double.class); + HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_THROUGHPUT_THRESHOLD_IN_BYTES, + DEFAULT_IO_TOTAL_THROUGHPUT_THRESHOLD_IN_BYTE_PER_SEC, (s) -> (s > 0), Double.class); ioTotSysCallRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME, - HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_SYSCALL_RATE_THRESHOLD_PER_SECOND, Double.class); - if (cpuUtilizationThreshold == null) { - cpuUtilizationThreshold = DEFAULT_CPU_UTILIZATION_THRESHOLD; - } - if (ioTotThroughputThreshold == null) { - ioTotThroughputThreshold = DEFAULT_IO_TOTAL_THROUGHPUT_THRESHOLD_IN_BYTE_PER_SEC; - } - if (ioTotSysCallRateThreshold == null) { - ioTotSysCallRateThreshold = DEFAULT_IO_TOTAL_SYSCALL_RATE_THRESHOLD_PER_SEC; - } + HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_SYSCALL_RATE_THRESHOLD_PER_SECOND, + DEFAULT_IO_TOTAL_SYSCALL_RATE_THRESHOLD_PER_SEC, (s) -> (s > 0), Double.class); } public double getCpuUtilizationThreshold() { diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java index 9189e5ed0..22209544b 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java @@ -24,10 +24,8 @@ public class QueueRejectionRcaConfig { public QueueRejectionRcaConfig(final RcaConf rcaConf) { rejectionTimePeriodInSeconds = rcaConf.readRcaConfig( - CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.REJECTION_TIME_PERIOD_IN_SECONDS, Integer.class); - if (rejectionTimePeriodInSeconds == null) { - rejectionTimePeriodInSeconds = DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS; - } + CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.REJECTION_TIME_PERIOD_IN_SECONDS, + DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS, (s) -> (s > 0), Integer.class); } public int getRejectionTimePeriodInSeconds() { diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/ShardRequestCacheRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/ShardRequestCacheRcaConfig.java new file mode 100644 index 000000000..1993db9e2 --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/ShardRequestCacheRcaConfig.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; + +/** + * config object to store rca config settings for ShardRequestCacheRca + */ +public class ShardRequestCacheRcaConfig { + public static final String CONFIG_NAME = "shard-request-cache-rca-config"; + + private Double shardRequestCacheSizeThreshold; + private Integer shardRequestCollectorTimePeriodInSec; + + // Shard request cache size threshold is 90% + public static final double DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD = 0.9; + // Metrics like eviction, hits are collected every 300 sec in shard request cache rca + public static final int DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC = 300; + + public ShardRequestCacheRcaConfig(final RcaConf rcaConf) { + shardRequestCacheSizeThreshold = + rcaConf.readRcaConfig( + CONFIG_NAME, + RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_CACHE_SIZE_THRESHOLD, + DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD, + (s) -> (s > 0), + Double.class); + shardRequestCollectorTimePeriodInSec = + rcaConf.readRcaConfig( + CONFIG_NAME, + RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC, + DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC, + (s) -> (s > 0), + Integer.class); + } + + public double getShardRequestCacheSizeThreshold() { + return shardRequestCacheSizeThreshold; + } + + public int getShardRequestCollectorTimePeriodInSec() { + return shardRequestCollectorTimePeriodInSec; + } + + public static class RCA_CONF_KEY_CONSTANTS { + public static final String SHARD_REQUEST_CACHE_SIZE_THRESHOLD = "shard-request-cache-threshold"; + public static final String SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC = "shard-request-collector-time-period-in-sec"; + } +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java index 193a95c43..2db7f29bc 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java @@ -20,14 +20,15 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.CacheActionConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.QueueActionConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.RcaControllerHelper; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.CacheConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.DeciderConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.FieldDataCacheRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageOldGenRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageYoungGenRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotNodeClusterRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardClusterRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.QueueRejectionRcaConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.ShardRequestCacheRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.RcaConsts; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; @@ -52,6 +53,8 @@ import java.util.Map; import java.util.Scanner; import java.util.Set; +import java.util.function.Predicate; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -175,8 +178,12 @@ public HotShardClusterRcaConfig getHotShardClusterRcaConfig() { return new HotShardClusterRcaConfig(this); } - public CacheConfig getCacheConfig() { - return new CacheConfig(this); + public FieldDataCacheRcaConfig getFieldDataCacheRcaConfig() { + return new FieldDataCacheRcaConfig(this); + } + + public ShardRequestCacheRcaConfig getShardRequestCacheRcaConfig() { + return new ShardRequestCacheRcaConfig(this); } public DeciderConfig getDeciderConfig() { @@ -206,22 +213,30 @@ public CacheActionConfig getCacheActionConfig() { public QueueActionConfig getQueueActionConfig() { return new QueueActionConfig(this); } - + + public T readRcaConfig(String rcaName, String key, T defaultValue, Class clazz) { + return readRcaConfig(rcaName, key, defaultValue, (s) -> true, clazz); + } + @SuppressWarnings("unchecked") - public T readRcaConfig(String rcaName, String key, Class clazz) { - T setting = null; + public T readRcaConfig(String rcaName, String key, T defaultValue, Predicate validator, Class clazz) { + T setting = defaultValue; try { Map rcaObj = null; if (conf.getRcaConfigSettings() != null - && conf.getRcaConfigSettings().containsKey(rcaName) - && conf.getRcaConfigSettings().get(rcaName) != null) { + && conf.getRcaConfigSettings().containsKey(rcaName) + && conf.getRcaConfigSettings().get(rcaName) != null) { rcaObj = (Map) conf.getRcaConfigSettings().get(rcaName); } if (rcaObj != null - && rcaObj.containsKey(key) - && rcaObj.get(key) != null) { + && rcaObj.containsKey(key) + && rcaObj.get(key) != null) { setting = clazz.cast(rcaObj.get(key)); + if (!validator.test(setting)) { + LOG.error("Config value: [{}] provided for key: [{}] is invalid", setting, key); + return defaultValue; + } } } catch (ClassCastException ne) { LOG.error("rca.conf contains value in invalid format, trace : {}", ne.getMessage()); diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/FieldDataCacheRca.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/FieldDataCacheRca.java index e55a04ad9..b22242670 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/FieldDataCacheRca.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/FieldDataCacheRca.java @@ -22,7 +22,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.FlowUnitMessage; import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.Resource; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.CacheConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.FieldDataCacheRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Rca; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Resources; @@ -73,7 +73,6 @@ */ public class FieldDataCacheRca extends Rca> { private static final Logger LOG = LogManager.getLogger(FieldDataCacheRca.class); - private static final long EVICTION_THRESHOLD_TIME_PERIOD_IN_MILLISECOND = TimeUnit.SECONDS.toMillis(300); private final Metric fieldDataCacheEvictions; private final Metric fieldDataCacheSizeGroupByOperation; @@ -93,10 +92,10 @@ public FieldDataCacheRca(final int rcaPeriod, this.fieldDataCacheEvictions = fieldDataCacheEvictions; this.fieldDataCacheSizeGroupByOperation = fieldDataCacheSizeGroupByOperation; this.counter = 0; - this.cacheSizeThreshold = CacheConfig.DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD; + this.cacheSizeThreshold = FieldDataCacheRcaConfig.DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD; this.clock = Clock.systemUTC(); this.cacheEvictionCollector = new CacheEvictionCollector(FIELD_DATA_CACHE_EVICTION, - fieldDataCacheEvictions, EVICTION_THRESHOLD_TIME_PERIOD_IN_MILLISECOND); + fieldDataCacheEvictions, FieldDataCacheRcaConfig.DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC); } @VisibleForTesting @@ -122,27 +121,29 @@ public ResourceFlowUnit operate() { if (cacheEvictionCollector.isUnhealthy(currTimestamp) && exceedsSizeThreshold) { context = new ResourceContext(Resources.State.UNHEALTHY); nodeSummary.appendNestedSummary(cacheEvictionCollector.generateSummary(currTimestamp)); - } - else { + } else { context = new ResourceContext(Resources.State.HEALTHY); } counter = 0; return new ResourceFlowUnit<>(currTimestamp, context, nodeSummary, !instanceDetails.getIsMaster()); - } - else { + } else { return new ResourceFlowUnit<>(currTimestamp); } } /** * read threshold values from rca.conf + * * @param conf RcaConf object */ @Override public void readRcaConf(RcaConf conf) { - CacheConfig configObj = conf.getCacheConfig(); + FieldDataCacheRcaConfig configObj = conf.getFieldDataCacheRcaConfig(); cacheSizeThreshold = configObj.getFieldDataCacheSizeThreshold(); + long cacheCollectorTimePeriodInSec = + TimeUnit.SECONDS.toMillis(configObj.getFieldDataCollectorTimePeriodInSec()); + cacheEvictionCollector.setCollectorTimePeriod(cacheCollectorTimePeriodInSec); } @Override @@ -165,15 +166,19 @@ private static class CacheEvictionCollector { private final Metric cacheEvictionMetrics; private boolean hasEvictions; private long evictionTimestamp; - private long evictionTimePeriodThreshold; + private long metricTimePeriodInMillis; private CacheEvictionCollector(final Resource cache, final Metric cacheEvictionMetrics, - final long threshold) { + final int metricTimePeriodInMillis) { this.cache = cache; this.cacheEvictionMetrics = cacheEvictionMetrics; this.hasEvictions = false; this.evictionTimestamp = 0; - this.evictionTimePeriodThreshold = threshold; + this.metricTimePeriodInMillis = TimeUnit.SECONDS.toMillis(metricTimePeriodInMillis); + } + + public void setCollectorTimePeriod(long metricTimePeriodInMillis) { + this.metricTimePeriodInMillis = metricTimePeriodInMillis; } public void collect(final long currTimestamp) { @@ -190,24 +195,22 @@ record -> record.getValue(MetricsDB.MAX, Double.class)).sum(); evictionTimestamp = currTimestamp; } hasEvictions = true; - } - else { + } else { hasEvictions = false; } - } - else { + } else { LOG.error("Failed to parse metric from cache {}", cache.toString()); } } } public boolean isUnhealthy(final long currTimestamp) { - return hasEvictions && (currTimestamp - evictionTimestamp) >= evictionTimePeriodThreshold; + return hasEvictions && (currTimestamp - evictionTimestamp) >= metricTimePeriodInMillis; } private HotResourceSummary generateSummary(final long currTimestamp) { return new HotResourceSummary(cache, - TimeUnit.MILLISECONDS.toSeconds(evictionTimePeriodThreshold), + TimeUnit.MILLISECONDS.toSeconds(metricTimePeriodInMillis), TimeUnit.MILLISECONDS.toSeconds(currTimestamp - evictionTimestamp), 0); } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/ShardRequestCacheRca.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/ShardRequestCacheRca.java index f09b3c6ae..fd2e389c9 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/ShardRequestCacheRca.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/ShardRequestCacheRca.java @@ -23,7 +23,8 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.FlowUnitMessage; import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.Resource; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.CacheConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.FieldDataCacheRcaConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.ShardRequestCacheRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Rca; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Resources; @@ -80,7 +81,6 @@ */ public class ShardRequestCacheRca extends Rca> { private static final Logger LOG = LogManager.getLogger(ShardRequestCacheRca.class); - private static final long THRESHOLD_TIME_PERIOD_IN_MILLISECOND = TimeUnit.SECONDS.toMillis(300); private final Metric shardRequestCacheEvictions; private final Metric shardRequestCacheHits; @@ -102,12 +102,18 @@ public ShardRequestCacheRca(final int rcaPeriod, this.shardRequestCacheHits = shardRequestCacheHits; this.shardRequestCacheSizeGroupByOperation = shardRequestCacheSizeGroupByOperation; this.counter = 0; - this.cacheSizeThreshold = CacheConfig.DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD; + this.cacheSizeThreshold = ShardRequestCacheRcaConfig.DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD; this.clock = Clock.systemUTC(); - this.cacheEvictionCollector = new CacheCollector(SHARD_REQUEST_CACHE_EVICTION, - shardRequestCacheEvictions, THRESHOLD_TIME_PERIOD_IN_MILLISECOND); - this.cacheHitCollector = new CacheCollector(SHARD_REQUEST_CACHE_HIT, - shardRequestCacheHits, THRESHOLD_TIME_PERIOD_IN_MILLISECOND); + this.cacheEvictionCollector = + new CacheCollector( + SHARD_REQUEST_CACHE_EVICTION, + shardRequestCacheEvictions, + ShardRequestCacheRcaConfig.DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC); + this.cacheHitCollector = + new CacheCollector( + SHARD_REQUEST_CACHE_HIT, + shardRequestCacheHits, + ShardRequestCacheRcaConfig.DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC); } @VisibleForTesting @@ -152,12 +158,17 @@ public ResourceFlowUnit operate() { /** * read threshold values from rca.conf + * * @param conf RcaConf object */ @Override public void readRcaConf(RcaConf conf) { - CacheConfig configObj = conf.getCacheConfig(); + ShardRequestCacheRcaConfig configObj = conf.getShardRequestCacheRcaConfig(); cacheSizeThreshold = configObj.getShardRequestCacheSizeThreshold(); + long cacheCollectorTimePeriodInSec = + TimeUnit.SECONDS.toMillis(configObj.getShardRequestCollectorTimePeriodInSec()); + cacheHitCollector.setCollectorTimePeriod(cacheCollectorTimePeriodInSec); + cacheEvictionCollector.setCollectorTimePeriod(cacheCollectorTimePeriodInSec); } @Override @@ -180,14 +191,18 @@ private static class CacheCollector { private final Metric cacheMetrics; private boolean hasMetric; private long metricTimestamp; - private long metricTimePeriodThreshold; + private long metricTimePeriodInMillis; - public CacheCollector(final Resource cache, final Metric cacheMetrics, final long threshold) { + public CacheCollector(final Resource cache, final Metric cacheMetrics, final int metricTimePeriodInSec) { this.cache = cache; this.cacheMetrics = cacheMetrics; this.hasMetric = false; this.metricTimestamp = 0; - this.metricTimePeriodThreshold = threshold; + this.metricTimePeriodInMillis = TimeUnit.SECONDS.toMillis(metricTimePeriodInSec); + } + + public void setCollectorTimePeriod(long metricTimePeriodInMillis) { + this.metricTimePeriodInMillis = metricTimePeriodInMillis; } public void collect(final long currTimestamp) { @@ -205,24 +220,22 @@ record -> record.getValue(MetricsDB.MAX, Double.class)).sum(); metricTimestamp = currTimestamp; } hasMetric = true; - } - else { + } else { hasMetric = false; } - } - else { + } else { LOG.error("Failed to parse metric from cache {}", cache.toString()); } } } public boolean isUnhealthy(final long currTimestamp) { - return hasMetric && (currTimestamp - metricTimestamp) >= metricTimePeriodThreshold; + return hasMetric && (currTimestamp - metricTimestamp) >= metricTimePeriodInMillis; } private HotResourceSummary generateSummary(final long currTimestamp) { return new HotResourceSummary(cache, - TimeUnit.MILLISECONDS.toSeconds(metricTimePeriodThreshold), + TimeUnit.MILLISECONDS.toSeconds(metricTimePeriodInMillis), TimeUnit.MILLISECONDS.toSeconds(currTimestamp - metricTimestamp), 0); } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConfTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConfTest.java index 7f4dd8700..fd85a84fc 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConfTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConfTest.java @@ -16,10 +16,12 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.DeciderConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.FieldDataCacheRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageOldGenRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageOldGenRcaConfig.RCA_CONF_KEY_CONSTANTS; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageYoungGenRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotNodeClusterRcaConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.ShardRequestCacheRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.decider.CachePriorityOrderConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.decider.WorkLoadTypeConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.RcaConsts; @@ -42,26 +44,53 @@ public void init() { @Test public void testReadRcaConfig() { - Integer topK = rcaConf.readRcaConfig(HighHeapUsageOldGenRcaConfig.CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.TOP_K, Integer.class); + Integer topK = rcaConf.readRcaConfig(HighHeapUsageOldGenRcaConfig.CONFIG_NAME, + RCA_CONF_KEY_CONSTANTS.TOP_K, HighHeapUsageOldGenRcaConfig.DEFAULT_TOP_K, Integer.class); Assert.assertNotNull(topK); Assert.assertEquals(HighHeapUsageOldGenRcaConfig.DEFAULT_TOP_K, topK.intValue()); Integer promotionRateThreshold = rcaConf.readRcaConfig(HighHeapUsageYoungGenRcaConfig.CONFIG_NAME, - HighHeapUsageYoungGenRcaConfig.RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, Integer.class); + HighHeapUsageYoungGenRcaConfig.RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, + HighHeapUsageYoungGenRcaConfig.DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC, Integer.class); Assert.assertNotNull(promotionRateThreshold); Assert.assertEquals(HighHeapUsageYoungGenRcaConfig.DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC, promotionRateThreshold.intValue()); Double unbalancedResourceThreshold = rcaConf.readRcaConfig(HotNodeClusterRcaConfig.CONFIG_NAME, - HotNodeClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, Double.class); + HotNodeClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, + HotNodeClusterRcaConfig.DEFAULT_UNBALANCED_RESOURCE_THRES, Double.class); Assert.assertNotNull(unbalancedResourceThreshold); Assert.assertEquals(HotNodeClusterRcaConfig.DEFAULT_UNBALANCED_RESOURCE_THRES, unbalancedResourceThreshold, 0.01); Integer val = rcaConf.readRcaConfig(HotNodeClusterRcaConfig.CONFIG_NAME, - HotNodeClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, Integer.class); - Assert.assertNull(val); + HotNodeClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, + 0, Integer.class); + Assert.assertNotNull(val); + Assert.assertEquals(0, val.intValue()); - val = rcaConf.readRcaConfig(HighHeapUsageOldGenRcaConfig.CONFIG_NAME, "test", Integer.class); - Assert.assertNull(val); + val = rcaConf.readRcaConfig(HighHeapUsageOldGenRcaConfig.CONFIG_NAME, "test", 0, Integer.class); + Assert.assertNotNull(val); + Assert.assertEquals(0, val.intValue()); + + Integer fieldDataTimePeriod = rcaConf.readRcaConfig(FieldDataCacheRcaConfig.CONFIG_NAME, + FieldDataCacheRcaConfig.RCA_CONF_KEY_CONSTANTS.FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC, + FieldDataCacheRcaConfig.DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC, Integer.class); + Assert.assertNotNull(fieldDataTimePeriod); + Assert.assertEquals(FieldDataCacheRcaConfig.DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC, fieldDataTimePeriod.intValue()); + + Integer shardRequestTimePeriod = rcaConf.readRcaConfig(ShardRequestCacheRcaConfig.CONFIG_NAME, + ShardRequestCacheRcaConfig.RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC, + ShardRequestCacheRcaConfig.DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC, Integer.class); + Assert.assertNotNull(shardRequestTimePeriod); + Assert.assertEquals(ShardRequestCacheRcaConfig.DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC, shardRequestTimePeriod.intValue()); + } + + @Test + public void testValidateRcaConfig() { + Integer defaultValue = rcaConf.readRcaConfig(ShardRequestCacheRcaConfig.CONFIG_NAME, + ShardRequestCacheRcaConfig.RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC, + 0, s -> s < 1, Integer.class); + Assert.assertNotNull(defaultValue); + Assert.assertEquals(0, defaultValue.intValue()); } @Test diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/RcaItCacheTuning.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/RcaItCacheTuning.java index 92391907f..15efbcd6d 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/RcaItCacheTuning.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/RcaItCacheTuning.java @@ -25,11 +25,13 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Request_Eviction; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Request_Hit; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Request_Size; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.RcaItMarker; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AClusterType; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AErrorPatternIgnored; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AExpect; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AMetric; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ARcaConf; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ARcaGraph; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATable; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATuple; @@ -39,6 +41,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.runners.RcaItNotEncryptedRunner; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning.validator.FieldDataCacheValidator; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.cache_tuning.validator.ShardRequestCacheValidator; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.queue_tuning.RcaItQueueTuning; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.ElasticSearchAnalysisGraph; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.FieldDataCacheClusterRca; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.ShardRequestCacheClusterRca; @@ -50,6 +53,8 @@ @RunWith(RcaItNotEncryptedRunner.class) @AClusterType(ClusterType.MULTI_NODE_CO_LOCATED_MASTER) @ARcaGraph(ElasticSearchAnalysisGraph.class) +//specify a custom rca.conf to set the collector time periods to 5s to reduce runtime +@ARcaConf(dataNode = RcaItCacheTuning.CACHE_TUNING_RESOURCES_DIR + "rca.conf") @AMetric( name = Cache_FieldData_Size.class, dimensionNames = { @@ -189,6 +194,7 @@ }) }) public class RcaItCacheTuning { + public static final String CACHE_TUNING_RESOURCES_DIR = Consts.INTEG_TESTS_SRC_DIR + "./tests/cache_tuning/resource/"; public static final String INDEX_NAME = "MockIndex"; public static final String SHARD_ID = "1"; @@ -217,6 +223,9 @@ public class RcaItCacheTuning { @AErrorPatternIgnored( pattern = "ModifyCacheMaxSizeAction:build()", reason = "Heap metrics is expected to be missing in this integ test.") + @AErrorPatternIgnored( + pattern = "SubscribeResponseHandler:onError()", + reason = "A unit test expressly calls SubscribeResponseHandler#onError, which writes an error log") public void testFieldDataCacheRca() {} // Test ShardRequestCacheClusterRca. @@ -243,5 +252,8 @@ public void testFieldDataCacheRca() {} @AErrorPatternIgnored( pattern = "ModifyCacheMaxSizeAction:build()", reason = "Heap metrics is expected to be missing in this integ test.") + @AErrorPatternIgnored( + pattern = "SubscribeResponseHandler:onError()", + reason = "A unit test expressly calls SubscribeResponseHandler#onError, which writes an error log") public void testShardRequestCacheRca() {} } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/resource/rca.conf b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/resource/rca.conf new file mode 100644 index 000000000..1a654569e --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/cache_tuning/resource/rca.conf @@ -0,0 +1,79 @@ +{ + "analysis-graph-implementor": + "com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.AnalysisGraphTest", + // it can be file:// or s3:// + "rca-store-location": "s3://sifi-store/rcas/", + + //it can be file:// or s3:// + "threshold-store-location": "s3://sifi-store/thresholds/", + + "new-rca-check-minutes": 60, + + "new-threshold-check-minutes": 30, + + // The size of the task queue for all networking operations. + // Small size queues may result in dropping of flow units, while large size queues can lead to a + // bigger backlog of tasks resulting in delays in sending and receiving. + "network-queue-length": 200, + + // The size of the per-vertex buffer for flow units received from remote nodes. + // Small buffer sizes may result in dropping of flow units, while large buffer size can lead to + // high memory consumptions depending on how the analysis graph is configured. + "max-flow-units-per-vertex-buffer": 200, + + "tags": { + "locus": "data-node", + "disk": "ssd", + "region": "use1", + "instance-type": "i3.8xl", + "domain": "rca-test-cluster" + }, + + "remote-peers": ["ip1", "ip2", "ip3"], + + // Tells the runtime where the RCAs will be stored. + "datastore": { + // accepted types are sqlite, in-memory. + "type": "sqlite", + "location-dir": "/tmp", + "filename": "rca.sqlite", + "storage-file-retention-count": 5, + // How often the sqlite file be repeated in seconds. This file contains RCAs and therefore rotating it too frequently + // might not be as fruitful as there might not be any data. + "rotation-period-seconds": 21600 + }, + + // Add config settings for different RCAs + "rca-config-settings": { + // old gen rca + "high-heap-usage-old-gen-rca": { + "top-k" : 3 + }, + //young gen rca + "high-heap-usage-young-gen-rca": { + "promotion-rate-mb-per-second" : 500, + "young-gen-gc-time-ms-per-second" : 400 + }, + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 5 + }, + //hot shard rca + "hot-shard-rca": { + "cpu-utilization" : 0.01, + "io-total-throughput-in-bytes" : 250000.0, + "io-total-syscallrate-per-second" : 0.1 + }, + "field-data-cache-rca-config": { + "field-data-cache-size-threshold" : 0.8, + "field-data-collector-time-period-in-sec" : 5 + }, + "shard-request-cache-rca-config": { + "shard-request-cache-threshold" : 0.9, + "shard-request-collector-time-period-in-sec" : 5 + } + }, + + "muted-rcas": [], + "muted-deciders": [], + "muted-actions": [] +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/RcaItQueueTuning.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/RcaItQueueTuning.java index 8f079f065..55a8e49ce 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/RcaItQueueTuning.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/RcaItQueueTuning.java @@ -107,6 +107,8 @@ public class RcaItQueueTuning { reason = "Cache related configs are expected to be missing in this integ test") @AErrorPatternIgnored(pattern = "AggregateMetric:gather()", reason = "Cache metrics are expected to be missing in this integ test") + @AErrorPatternIgnored(pattern = "SubscribeResponseHandler:onError()", + reason = "A unit test expressly calls SubscribeResponseHandler#onError, which writes an error log") public void testQueueRejectionRca() { } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/resource/rca.conf b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/resource/rca.conf index 00a05f29e..33efa0ecd 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/resource/rca.conf +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/queue_tuning/resource/rca.conf @@ -70,9 +70,5 @@ "muted-actions": [], "decider-config-settings": { - "cache-decider-config": { - "field-data-cache-upper-bound" : 0.4, - "shard-request-cache-upper-bound" : 0.05 - } } } \ No newline at end of file diff --git a/src/test/resources/rca/rca_elected_master.conf b/src/test/resources/rca/rca_elected_master.conf index a840eb2a5..54cc22a64 100644 --- a/src/test/resources/rca/rca_elected_master.conf +++ b/src/test/resources/rca/rca_elected_master.conf @@ -73,6 +73,14 @@ "cpu-utilization-cluster-percentage" : 0.3, "io-total-throughput-cluster-percentage" : 0.3, "io-total-syscallrate-cluster-percentage" : 0.3 + }, + "field-data-cache-rca-config": { + "field-data-cache-size-threshold" : 0.8, + "field-data-collector-time-period-in-sec" : 300 + }, + "shard-request-cache-rca-config": { + "shard-request-cache-threshold" : 0.9, + "shard-request-collector-time-period-in-sec" : 300 } },