Add support for action configs

pa_config/rca.conf

@@ -103,11 +103,32 @@
     // Priority order in the list goes from most used to the lease used cache type.
     "cache-type": {
       "priority-order": ["fielddata-cache", "shard-request-cache", "query-cache", "bitset-filter-cache"]
-    },
-    // cache decider - Needs to be updated as per the performance test results
-    "cache-bounds": {
-      "field-data-cache-upper-bound" : 0.4,
-      "shard-request-cache-upper-bound" : 0.05
     }
+  },
+
+  // Action Configurations
+  "action-config-settings": {
+//    // Cache Max Size bounds are expressed as %age of JVM heap size
+//    "cache-settings": {
+//      "fielddata": {
+//        "upper-bound": 0.4,
+//        "lower-bound": 0.1
+//      },
+//      "shard-request": {
+//        "upper-bound": 0.05,
+//        "lower-bound": 0.01
+//      }
+//    },
+//    // Queue Capacity bounds are expressed as absolute queue size
+//    "queue-settings": {
+//      "search": {
+//        "upper-bound": 3000,
+//        "lower-bound": 1000
+//      },
+//      "write": {
+//        "upper-bound": 1000,
+//        "lower-bound": 50
+//      }
+//    }
   }
 }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/actions/ModifyCacheMaxSizeAction.java

@@ -20,9 +20,12 @@
 import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache.CacheUtil.MB_TO_BYTES;
 
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.AppContext;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.CacheActionConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.ResourceEnum;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.util.NodeConfigCacheReaderUtil;
+import com.google.common.annotations.VisibleForTesting;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -40,7 +43,6 @@ public class ModifyCacheMaxSizeAction extends SuppressibleAction {
 
   private final NodeKey esNode;
   private final ResourceEnum cacheType;
-  private final AppContext appContext;
 
   private final long desiredCacheMaxSizeInBytes;
   private final long currentCacheMaxSizeInBytes;
@@ -58,8 +60,6 @@ public ModifyCacheMaxSizeAction(
     super(appContext);
     this.esNode = esNode;
     this.cacheType = cacheType;
-    this.appContext = appContext;
-
     this.desiredCacheMaxSizeInBytes = desiredCacheMaxSizeInBytes;
     this.currentCacheMaxSizeInBytes = currentCacheMaxSizeInBytes;
     this.coolOffPeriodInMillis = coolOffPeriodInMillis;
@@ -70,8 +70,8 @@ public static Builder newBuilder(
       final NodeKey esNode,
       final ResourceEnum cacheType,
       final AppContext appContext,
-      double upperBoundThreshold) {
-    return new Builder(esNode, cacheType, appContext, upperBoundThreshold);
+      final RcaConf conf) {
+    return new Builder(esNode, cacheType, appContext, conf);
   }
 
   @Override
@@ -135,6 +135,11 @@ public ResourceEnum getCacheType() {
     return cacheType;
   }
 
+  @VisibleForTesting
+  public static long getThresholdInBytes(double threshold, long heapSize) {
+    return (long) (threshold * heapSize);
+  }
+
   public static final class Builder {
     public static final long DEFAULT_COOL_OFF_PERIOD_IN_MILLIS = 300 * 1_000;
     public static final boolean DEFAULT_IS_INCREASE = true;
@@ -143,7 +148,7 @@ public static final class Builder {
     private final ResourceEnum cacheType;
     private final NodeKey esNode;
     private final AppContext appContext;
-    private double upperBoundThreshold;
+    private final RcaConf rcaConf;
 
     private long stepSizeInBytes;
     private boolean isIncrease;
@@ -153,34 +158,46 @@ public static final class Builder {
     private Long currentCacheMaxSizeInBytes;
     private Long desiredCacheMaxSizeInBytes;
     private Long heapMaxSizeInBytes;
+    private final long upperBoundInBytes;
+    private final long lowerBoundInBytes;
 
     private Builder(
         final NodeKey esNode,
         final ResourceEnum cacheType,
         final AppContext appContext,
-        double upperBoundThreshold) {
+        final RcaConf conf) {
       this.esNode = esNode;
       this.cacheType = cacheType;
       this.appContext = appContext;
-      this.upperBoundThreshold = upperBoundThreshold;
+      this.rcaConf = conf;
 
       this.coolOffPeriodInMillis = DEFAULT_COOL_OFF_PERIOD_IN_MILLIS;
       this.isIncrease = DEFAULT_IS_INCREASE;
       this.canUpdate = DEFAULT_CAN_UPDATE;
 
-      this.currentCacheMaxSizeInBytes =
-          NodeConfigCacheReaderUtil.readCacheMaxSizeInBytes(
-              appContext.getNodeConfigCache(), esNode, cacheType);
-      this.heapMaxSizeInBytes =
-          NodeConfigCacheReaderUtil.readHeapMaxSizeInBytes(appContext.getNodeConfigCache(), esNode);
+      this.currentCacheMaxSizeInBytes = NodeConfigCacheReaderUtil.readCacheMaxSizeInBytes(
+          appContext.getNodeConfigCache(), esNode, cacheType);
+      this.heapMaxSizeInBytes = NodeConfigCacheReaderUtil.readHeapMaxSizeInBytes(
+          appContext.getNodeConfigCache(), esNode);
       this.desiredCacheMaxSizeInBytes = null;
       setDefaultStepSize(cacheType);
+
+      CacheActionConfig cacheActionConfig = new CacheActionConfig(rcaConf);
+      double upperBoundThreshold = cacheActionConfig.getThresholdConfig(cacheType).upperBound();
+      double lowerBoundThreshold = cacheActionConfig.getThresholdConfig(cacheType).lowerBound();
+      if (heapMaxSizeInBytes != null) {
+        this.upperBoundInBytes = getThresholdInBytes(upperBoundThreshold, heapMaxSizeInBytes);
+        this.lowerBoundInBytes = getThresholdInBytes(lowerBoundThreshold, heapMaxSizeInBytes);
+      } else {
+        // If heapMaxSizeInBytes is null, we return a non-actionable object from build
+        this.upperBoundInBytes = 0;
+        this.lowerBoundInBytes = 0;
+      }
     }
 
     private void setDefaultStepSize(ResourceEnum cacheType) {
       // TODO: Move configuration values to rca.conf
-      // TODO: Update the step size to also include percentage of heap size along with absolute
-      // value
+      // TODO: Update the step size to also include percentage of heap size along with absolute value
       switch (cacheType) {
         case FIELD_DATA_CACHE:
           // Field data cache having step size of 512MB
@@ -191,8 +208,7 @@ private void setDefaultStepSize(ResourceEnum cacheType) {
           this.stepSizeInBytes = (long) 512 * KB_TO_BYTES;
           break;
         default:
-          throw new IllegalArgumentException(
-                String.format("Unrecognizable cache type: [%s]", cacheType.toString()));
+          throw new IllegalArgumentException(String.format("Unrecognizable cache type: [%s]", cacheType.toString()));
       }
     }
 
@@ -206,48 +222,39 @@ public Builder increase(boolean isIncrease) {
       return this;
     }
 
-    public Builder desiredCacheMaxSize(long desiredCacheMaxSizeInBytes) {
-      this.desiredCacheMaxSizeInBytes = desiredCacheMaxSizeInBytes;
+    public Builder setDesiredCacheMaxSizeToMin() {
+      this.desiredCacheMaxSizeInBytes = lowerBoundInBytes;
       return this;
     }
 
-    public Builder stepSizeInBytes(long stepSizeInBytes) {
-      this.stepSizeInBytes = stepSizeInBytes;
+    public Builder setDesiredCacheMaxSizeToMax() {
+      this.desiredCacheMaxSizeInBytes = upperBoundInBytes;
       return this;
     }
 
-    public Builder upperBoundThreshold(double upperBoundThreshold) {
-      this.upperBoundThreshold = upperBoundThreshold;
+    public Builder stepSizeInBytes(long stepSizeInBytes) {
+      this.stepSizeInBytes = stepSizeInBytes;
       return this;
     }
 
     public ModifyCacheMaxSizeAction build() {
-      // In case of failure to read cache max size or heap max size from node config cache
-      // return an empty non-actionable action object
       if (currentCacheMaxSizeInBytes == null || heapMaxSizeInBytes == null) {
-        LOG.error(
-            "Action: Fail to read cache max size or heap max size from node config cache. Return an non-actionable action");
-        return new ModifyCacheMaxSizeAction(
-            esNode, cacheType, appContext, -1, -1, coolOffPeriodInMillis, false);
+        LOG.error("Action: Fail to read cache max size or heap max size from node config cache. "
+            + "Return an non-actionable action");
+        return new ModifyCacheMaxSizeAction(esNode, cacheType, appContext,
+            -1, -1, coolOffPeriodInMillis, false);
       }
-      // skip the step size bound check if we set desiredCapacity
-      // explicitly in action builder
+
       if (desiredCacheMaxSizeInBytes == null) {
-        desiredCacheMaxSizeInBytes = currentCacheMaxSizeInBytes;
-        if (isIncrease) {
-          desiredCacheMaxSizeInBytes += stepSizeInBytes;
-        }
+        desiredCacheMaxSizeInBytes = isIncrease ? currentCacheMaxSizeInBytes + stepSizeInBytes :
+            currentCacheMaxSizeInBytes - stepSizeInBytes;
       }
-      long upperBoundInBytes = (long) (upperBoundThreshold * heapMaxSizeInBytes);
-      desiredCacheMaxSizeInBytes = Math.min(desiredCacheMaxSizeInBytes, upperBoundInBytes);
-      return new ModifyCacheMaxSizeAction(
-          esNode,
-          cacheType,
-          appContext,
-          desiredCacheMaxSizeInBytes,
-          currentCacheMaxSizeInBytes,
-          coolOffPeriodInMillis,
-          canUpdate);
+
+      // Ensure desired cache max size is within thresholds
+      desiredCacheMaxSizeInBytes = Math.max(Math.min(desiredCacheMaxSizeInBytes, upperBoundInBytes), lowerBoundInBytes);
+
+      return new ModifyCacheMaxSizeAction(esNode, cacheType, appContext,
+          desiredCacheMaxSizeInBytes, currentCacheMaxSizeInBytes, coolOffPeriodInMillis, canUpdate);
     }
   }
 }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/actions/ModifyQueueCapacityAction.java

@@ -20,7 +20,10 @@
 import static com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.ImpactVector.Dimension.NETWORK;
 
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.AppContext;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.QueueActionConfig;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.ResourceEnum;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Rca;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey;
 
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.util.NodeConfigCacheReaderUtil;
@@ -39,27 +42,22 @@ public class ModifyQueueCapacityAction extends SuppressibleAction {
   private final NodeKey esNode;
   private final int desiredCapacity;
   private final int currentCapacity;
-  private final int lowerBound;
-  private final int upperBound;
   private final long coolOffPeriodInMillis;
   private final boolean canUpdate;
 
   private ModifyQueueCapacityAction(NodeKey esNode, ResourceEnum threadPool, AppContext appContext,
-      int desiredCapacity, int currentCapacity, long coolOffPeriodInMillis,
-      int lowerBound, int upperBound, boolean canUpdate) {
+      int desiredCapacity, int currentCapacity, long coolOffPeriodInMillis, boolean canUpdate) {
     super(appContext);
     this.esNode = esNode;
     this.threadPool = threadPool;
     this.desiredCapacity = desiredCapacity;
     this.currentCapacity = currentCapacity;
     this.coolOffPeriodInMillis = coolOffPeriodInMillis;
-    this.lowerBound = lowerBound;
-    this.upperBound = upperBound;
     this.canUpdate = canUpdate;
   }
 
-  public static Builder newBuilder(NodeKey esNode, ResourceEnum threadPool, AppContext appContext) {
-    return new Builder(esNode, threadPool, appContext);
+  public static Builder newBuilder(NodeKey esNode, ResourceEnum threadPool, final AppContext appContext, final RcaConf conf) {
+    return new Builder(esNode, threadPool, appContext, conf);
   }
 
   @Override
@@ -129,43 +127,32 @@ public static final class Builder {
     private boolean increase;
     private boolean canUpdate;
     private long coolOffPeriodInMillis;
-    private int upperBound;
-    private int lowerBound;
     private final ResourceEnum threadPool;
     private final NodeKey esNode;
     private final AppContext appContext;
+    private final RcaConf rcaConf;
 
     private Integer currentCapacity;
     private Integer desiredCapacity;
+    private final int upperBound;
+    private final int lowerBound;
 
-    public Builder(NodeKey esNode, ResourceEnum threadPool, AppContext appContext) {
+    public Builder(NodeKey esNode, ResourceEnum threadPool, final AppContext appContext, final RcaConf conf) {
       this.esNode = esNode;
       this.threadPool = threadPool;
       this.appContext = appContext;
+      this.rcaConf = conf;
       this.coolOffPeriodInMillis = DEFAULT_COOL_OFF_PERIOD_IN_MILLIS;
       this.stepSize = DEFAULT_STEP_SIZE;
       this.increase = DEFAULT_IS_INCREASE;
       this.canUpdate = DEFAULT_CAN_UPDATE;
       this.desiredCapacity = null;
-      this.currentCapacity =
-          NodeConfigCacheReaderUtil.readQueueCapacity(appContext.getNodeConfigCache(), esNode, threadPool);
-      setDefaultBounds(threadPool);
-    }
+      this.currentCapacity = NodeConfigCacheReaderUtil.readQueueCapacity(
+          appContext.getNodeConfigCache(), esNode, threadPool);
 
-    private void setDefaultBounds(ResourceEnum threadPool) {
-      // TODO: Move configuration values to rca.conf
-      switch (threadPool) {
-        case WRITE_THREADPOOL:
-          this.lowerBound = 100;
-          this.upperBound = 1000;
-          break;
-        case SEARCH_THREADPOOL:
-          this.lowerBound = 1000;
-          this.upperBound = 3000;
-          break;
-        default:
-          assert false : "unrecognized threadpool type: " + threadPool.name();
-      }
+      QueueActionConfig queueActionConfig = new QueueActionConfig(rcaConf);
+      this.upperBound = queueActionConfig.getThresholdConfig(threadPool).upperBound();
+      this.lowerBound = queueActionConfig.getThresholdConfig(threadPool).lowerBound();
     }
 
     public Builder coolOffPeriod(long coolOffPeriodInMillis) {
@@ -178,13 +165,13 @@ public Builder increase(boolean increase) {
       return this;
     }
 
-    public Builder desiredCapacity(int desiredCapacity) {
-      this.desiredCapacity = desiredCapacity;
+    public Builder setDesiredCapacityToMin() {
+      this.desiredCapacity = this.lowerBound;
       return this;
     }
 
-    public Builder minimalDesiredCapacity() {
-      this.desiredCapacity = this.lowerBound;
+    public Builder setDesiredCapacityToMax() {
+      this.desiredCapacity = this.upperBound;
       return this;
     }
 
@@ -193,33 +180,21 @@ public Builder stepSize(int stepSize) {
       return this;
     }
 
-    public Builder upperBound(int upperBound) {
-      this.upperBound = upperBound;
-      return this;
-    }
-
-    public Builder lowerBound(int lowerBound) {
-      this.lowerBound = lowerBound;
-      return this;
-    }
-
     public ModifyQueueCapacityAction build() {
-      // fail to read capacity from node config cache
-      // return an empty non-actionable action object
       if (currentCapacity == null) {
         LOG.error("Action: Fail to read queue capacity from node config cache. Return an non-actionable action");
         return new ModifyQueueCapacityAction(esNode, threadPool, appContext,
-            -1, -1, coolOffPeriodInMillis, lowerBound, upperBound, false);
+            -1, -1, coolOffPeriodInMillis, false);
       }
-      // skip the step size bound check if we set desiredCapacity
-      // explicitly in action builder
       if (desiredCapacity == null) {
         desiredCapacity = increase ? currentCapacity + stepSize : currentCapacity - stepSize;
       }
+
+      // Ensure desired capacity is within configured safety bounds
       desiredCapacity = Math.min(desiredCapacity, upperBound);
       desiredCapacity = Math.max(desiredCapacity, lowerBound);
       return new ModifyQueueCapacityAction(esNode, threadPool, appContext,
-          desiredCapacity, currentCapacity, coolOffPeriodInMillis, lowerBound, upperBound, canUpdate);
+          desiredCapacity, currentCapacity, coolOffPeriodInMillis, canUpdate);
     }
   }
 }

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/actions/SuppressibleAction.java

@@ -7,7 +7,7 @@
  */
 public abstract class SuppressibleAction implements Action {
 
-  private final AppContext appContext;
+  protected final AppContext appContext;
 
   public SuppressibleAction(final AppContext appContext) {
     this.appContext = appContext;