diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index a7c1a00d..d6dd42b2 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -15,8 +15,11 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler.ConfigOverridesClusterSettingHandler; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CacheConfigMetricsCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler.NodeStatsSettingHandler; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config.PerformanceAnalyzerOverridesClusterConfigAction; import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config.PerformanceAnalyzerResourceProvider; import java.io.File; import java.security.AccessController; @@ -95,6 +98,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.transport.PerformanceAnalyzerTransportInterceptor; import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.Utils; import com.amazon.opendistro.elasticsearch.performanceanalyzer.writer.EventLogQueueProcessor; + import static java.util.Collections.singletonList; public final class PerformanceAnalyzerPlugin extends Plugin implements ActionPlugin, NetworkPlugin, SearchPlugin { @@ -104,6 +108,8 @@ public final class PerformanceAnalyzerPlugin extends Plugin implements ActionPlu private static SecurityManager sm = null; private final PerformanceAnalyzerClusterSettingHandler perfAnalyzerClusterSettingHandler; private final NodeStatsSettingHandler nodeStatsSettingHandler; + private final ConfigOverridesClusterSettingHandler configOverridesClusterSettingHandler; + private final ConfigOverridesWrapper configOverridesWrapper; private final PerformanceAnalyzerController performanceAnalyzerController; private final ClusterSettingsManager clusterSettingsManager; @@ -152,9 +158,27 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa //initialize plugin settings. Accessing plugin settings before this //point will break, as the plugin location will not be initialized. PluginSettings.instance(); - scheduledMetricCollectorsExecutor = new ScheduledMetricCollectorsExecutor(); this.performanceAnalyzerController = new PerformanceAnalyzerController(scheduledMetricCollectorsExecutor); + + configOverridesWrapper = new ConfigOverridesWrapper(); + clusterSettingsManager = new ClusterSettingsManager(Arrays.asList(PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, + PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING), + Collections.singletonList(PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING)); + configOverridesClusterSettingHandler = new ConfigOverridesClusterSettingHandler(configOverridesWrapper, clusterSettingsManager, + PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING); + clusterSettingsManager.addSubscriberForStringSetting(PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING, + configOverridesClusterSettingHandler); + perfAnalyzerClusterSettingHandler = new PerformanceAnalyzerClusterSettingHandler(performanceAnalyzerController, + clusterSettingsManager); + clusterSettingsManager.addSubscriberForIntSetting(PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, + perfAnalyzerClusterSettingHandler); + + nodeStatsSettingHandler = new NodeStatsSettingHandler(performanceAnalyzerController, + clusterSettingsManager); + clusterSettingsManager.addSubscriberForIntSetting(PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING, + nodeStatsSettingHandler); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ThreadPoolMetricsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new CacheConfigMetricsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new CircuitBreakerCollector()); @@ -163,7 +187,7 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MetricsPurgeActivity()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NodeDetailsCollector()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NodeDetailsCollector(configOverridesWrapper)); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NodeStatsMetricsCollector(performanceAnalyzerController)); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceMetrics()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceEventMetrics()); @@ -172,22 +196,6 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa scheduledMetricCollectorsExecutor.addScheduledMetricCollector(StatsCollector.instance()); scheduledMetricCollectorsExecutor.start(); - clusterSettingsManager = new ClusterSettingsManager( - Arrays.asList(PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, - PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING)); - - perfAnalyzerClusterSettingHandler = new PerformanceAnalyzerClusterSettingHandler( - performanceAnalyzerController, - clusterSettingsManager); - clusterSettingsManager.addSubscriberForSetting(PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, - perfAnalyzerClusterSettingHandler); - - nodeStatsSettingHandler = new NodeStatsSettingHandler( - performanceAnalyzerController, - clusterSettingsManager); - clusterSettingsManager.addSubscriberForSetting(PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING, - nodeStatsSettingHandler); - EventLog eventLog = new EventLog(); EventLogFileHandler eventLogFileHandler = new EventLogFileHandler(eventLog, PluginSettings.instance().getMetricsLocation()); new EventLogQueueProcessor(eventLogFileHandler, @@ -237,7 +245,10 @@ public List getRestHandlers(final Settings s PerformanceAnalyzerResourceProvider performanceAnalyzerRp = new PerformanceAnalyzerResourceProvider(settings, restController); PerformanceAnalyzerClusterConfigAction paClusterConfigAction = new PerformanceAnalyzerClusterConfigAction(settings, restController, perfAnalyzerClusterSettingHandler, nodeStatsSettingHandler); - return Arrays.asList(performanceanalyzerConfigAction, paClusterConfigAction, performanceAnalyzerRp); + PerformanceAnalyzerOverridesClusterConfigAction paOverridesConfigClusterAction = + new PerformanceAnalyzerOverridesClusterConfigAction(settings, restController, + configOverridesClusterSettingHandler, configOverridesWrapper); + return Arrays.asList(performanceanalyzerConfigAction, paClusterConfigAction, performanceAnalyzerRp, paOverridesConfigClusterAction); } @Override @@ -275,7 +286,8 @@ public Map> getTransports(Settings settings, ThreadP @Override public List> getSettings() { return Arrays.asList(PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, - PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING); + PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING, + PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING); } } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollector.java index 378ca3d2..0451f32d 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollector.java @@ -16,6 +16,8 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesHelper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.NodeDetailColumns; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.NodeRole; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; @@ -27,15 +29,22 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import java.io.IOException; import java.util.Iterator; public class NodeDetailsCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(NodeDetailsCollector.class).samplingInterval; private static final Logger LOG = LogManager.getLogger(NodeDetailsCollector.class); private static final int KEYS_PATH_LENGTH = 0; + private final ConfigOverridesWrapper configOverridesWrapper; public NodeDetailsCollector() { + this(null); + } + + public NodeDetailsCollector(final ConfigOverridesWrapper configOverridesWrapper) { super(SAMPLING_TIME_INTERVAL, "NodeDetails"); + this.configOverridesWrapper = configOverridesWrapper; } @Override @@ -54,6 +63,22 @@ public void collectMetrics(long startTime) { .append( PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + // We add the config overrides in line#2 because we don't know how many lines + // follow that belong to actual node details, and the reader also has no way to + // know this information in advance unless we add the number of nodes as + // additional metadata in the file. + try { + String rcaOverrides = ConfigOverridesHelper.serialize(configOverridesWrapper.getCurrentClusterConfigOverrides()); + value.append(rcaOverrides); + } catch (IOException ioe) { + LOG.error("Unable to serialize rca config overrides.", ioe); + } + value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + + // line#3 denotes when the timestamp when the config override happened. + value.append(configOverridesWrapper.getLastUpdatedTimestamp()); + value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + DiscoveryNodes discoveryNodes = ESResources.INSTANCE.getClusterService().state().nodes(); DiscoveryNode masterNode = discoveryNodes.getMasterNode(); diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/ClusterSettingsManager.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/ClusterSettingsManager.java index 3c9c825e..90666433 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/ClusterSettingsManager.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/ClusterSettingsManager.java @@ -28,14 +28,17 @@ */ public class ClusterSettingsManager implements ClusterStateListener { private static final Logger LOG = LogManager.getLogger(ClusterSettingsManager.class); - private Map, List>> listenerMap = new HashMap<>(); - private final List> managedSettings = new ArrayList<>(); + private final Map, List>> intSettingListenerMap = new HashMap<>(); + private final Map, List>> stringSettingListenerMap = new HashMap<>(); + private final List> managedIntSettings = new ArrayList<>(); + private final List> managedStringSettings = new ArrayList<>(); private final ClusterSettingsResponseHandler clusterSettingsResponseHandler; private boolean initialized = false; - public ClusterSettingsManager(List> initialSettings) { - managedSettings.addAll(initialSettings); + public ClusterSettingsManager(List> intSettings, List> stringSettings) { + managedIntSettings.addAll(intSettings); + managedStringSettings.addAll(stringSettings); this.clusterSettingsResponseHandler = new ClusterSettingsResponseHandler(); } @@ -45,18 +48,35 @@ public ClusterSettingsManager(List> initialSettings) { * @param setting The setting that needs to be listened to. * @param listener The listener object that will be called when the setting changes. */ - public void addSubscriberForSetting(Setting setting, ClusterSettingListener listener) { - if (listenerMap.containsKey(setting)) { - final List> currentListeners = listenerMap.get(setting); + public void addSubscriberForIntSetting(Setting setting, ClusterSettingListener listener) { + if (intSettingListenerMap.containsKey(setting)) { + final List> currentListeners = intSettingListenerMap.get(setting); if (!currentListeners.contains(listener)) { currentListeners.add(listener); - listenerMap.put(setting, currentListeners); + intSettingListenerMap.put(setting, currentListeners); } } else { - listenerMap.put(setting, Collections.singletonList(listener)); + intSettingListenerMap.put(setting, Collections.singletonList(listener)); } } + /** + * Adds a listener that will be called when the requested setting's value changes. + * + * @param setting The setting that needs to be listened to. + * @param listener The listener object that will be called when the setting changes. + */ + public void addSubscriberForStringSetting(Setting setting, ClusterSettingListener listener) { + if (stringSettingListenerMap.containsKey(setting)) { + final List> currentListeners = stringSettingListenerMap.get(setting); + if (!currentListeners.contains(listener)) { + currentListeners.add(listener); + stringSettingListenerMap.put(setting, currentListeners); + } + } else { + stringSettingListenerMap.put(setting, Collections.singletonList(listener)); + } + } /** * Bootstraps the listeners and tries to read initial values for cluster settings. */ @@ -91,14 +111,34 @@ public void updateSetting(final Setting setting, final Integer newValue ESResources.INSTANCE.getClient().admin().cluster().updateSettings(request); } + /** + * Updates the requested setting with the requested value across the cluster. + * + * @param setting The setting that needs to be updated. + * @param newValue The new value for the setting. + */ + public void updateSetting(final Setting setting, final String newValue) { + final ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); + request.persistentSettings(Settings.builder() + .put(setting.getKey(), newValue) + .build()); + ESResources.INSTANCE.getClient().admin().cluster().updateSettings(request); + } + /** * Registers a setting update listener for all the settings managed by this instance. */ private void registerSettingUpdateListener() { - for (Setting setting : managedSettings) { + for (Setting setting : managedIntSettings) { ESResources.INSTANCE.getClusterService() .getClusterSettings() - .addSettingsUpdateConsumer(setting, updatedVal -> callListeners(setting, updatedVal)); + .addSettingsUpdateConsumer(setting, updatedVal -> callIntSettingListeners(setting, updatedVal)); + } + + for (Setting setting : managedStringSettings) { + ESResources.INSTANCE.getClusterService() + .getClusterSettings() + .addSettingsUpdateConsumer(setting, updatedVal -> callStringSettingListeners(setting, updatedVal)); } } @@ -166,9 +206,9 @@ public void clusterChanged(final ClusterChangedEvent event) { * @param setting The setting whose listeners need to be notified. * @param settingValue The new value for the setting. */ - private void callListeners(final Setting setting, int settingValue) { + private void callIntSettingListeners(final Setting setting, int settingValue) { try { - final List> listeners = listenerMap.get(setting); + final List> listeners = intSettingListenerMap.get(setting); if (listeners != null) { for (ClusterSettingListener listener : listeners) { listener.onSettingUpdate(settingValue); @@ -180,6 +220,25 @@ private void callListeners(final Setting setting, int settingValue) { } } + /** + * Calls all the listeners for the specified setting with the requested value. + * + * @param setting The setting whose listeners need to be notified. + * @param settingValue The new value for the setting. + */ + private void callStringSettingListeners(final Setting setting, String settingValue) { + try { + final List> listeners = stringSettingListenerMap.get(setting); + if (listeners != null) { + for (ClusterSettingListener listener : listeners) { + listener.onSettingUpdate(settingValue); + } + } + } catch(Exception ex) { + LOG.error(ex); + StatsCollector.instance().logException(StatExceptionCode.ES_REQUEST_INTERCEPTOR_ERROR); + } + } /** * Class that handles response to GET /_cluster/settings */ @@ -196,10 +255,17 @@ public void onResponse(final ClusterStateResponse clusterStateResponse) { .getMetaData() .persistentSettings(); - for (final Setting setting : managedSettings) { + for (final Setting setting : managedIntSettings) { Integer settingValue = clusterSettings.getAsInt(setting.getKey(), null); if (settingValue != null) { - callListeners(setting, settingValue); + callIntSettingListeners(setting, settingValue); + } + } + + for (final Setting setting : managedStringSettings) { + String settingValue = clusterSettings.get(setting.getKey(), ""); + if (settingValue != null) { + callStringSettingListeners(setting, settingValue); } } } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java index e60c5e5e..a8e27b58 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java @@ -14,7 +14,7 @@ public final class PerformanceAnalyzerClusterSettings { 0, Setting.Property.NodeScope, Setting.Property.Dynamic - ); + ); public enum PerformanceAnalyzerFeatureBits { PA_BIT, @@ -33,4 +33,15 @@ public enum PerformanceAnalyzerFeatureBits { Setting.Property.NodeScope, Setting.Property.Dynamic ); + + /** + * Cluster setting controlling the config overrides to be applied on performance + * analyzer components. + */ + public static final Setting CONFIG_OVERRIDES_SETTING = Setting.simpleString( + "cluster.metadata.perf_analyzer.config.overrides", + "", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/ConfigOverridesClusterSettingHandler.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/ConfigOverridesClusterSettingHandler.java new file mode 100644 index 00000000..3b2a4b9d --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/ConfigOverridesClusterSettingHandler.java @@ -0,0 +1,201 @@ +/* + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverrides; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesHelper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.ClusterSettingListener; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.ClusterSettingsManager; +import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Setting; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +public class ConfigOverridesClusterSettingHandler implements ClusterSettingListener { + + private static final Logger LOG = LogManager.getLogger(ConfigOverridesClusterSettingHandler.class); + + private final ClusterSettingsManager clusterSettingsManager; + private final ConfigOverridesWrapper overridesHolder; + private final Setting setting; + + public ConfigOverridesClusterSettingHandler(final ConfigOverridesWrapper overridesHolder, + final ClusterSettingsManager clusterSettingsManager, + final Setting setting) { + this.clusterSettingsManager = clusterSettingsManager; + this.overridesHolder = overridesHolder; + this.setting = setting; + } + + /** + * Handler that gets called when there is a new value for the setting that this listener is + * listening to. + * + * @param newSettingValue The value of the new setting. + */ + @Override + public void onSettingUpdate(String newSettingValue) { + try { + final ConfigOverrides newOverrides = ConfigOverridesHelper.deserialize(newSettingValue); + overridesHolder.setCurrentClusterConfigOverrides(newOverrides); + overridesHolder.setLastUpdatedTimestamp(System.currentTimeMillis()); + } catch (IOException e) { + LOG.error("Unable to apply received cluster setting update: " + newSettingValue, e); + } + } + + /** + * Updates the cluster setting with the new set of config overrides. + * + * @param newOverrides The new set of overrides that need to be applied. + * @throws IOException if unable to serialize the setting. + */ + public void updateConfigOverrides(final ConfigOverrides newOverrides) throws IOException { + String newClusterSettingValue = buildClusterSettingValue(newOverrides); + // TODO: @ktkrg - Change to debug + LOG.error("Updating cluster setting with new overrides string: {}", newClusterSettingValue); + clusterSettingsManager.updateSetting(setting, newClusterSettingValue); + } + + /** + * Generates a string representation of overrides. + * + * @param newOverrides The new overrides that need to be merged with the existing + * overrides. + * @return String value of the merged config overrides. + */ + private String buildClusterSettingValue(final ConfigOverrides newOverrides) throws IOException { + final ConfigOverrides mergedConfigOverrides = merge(overridesHolder.getCurrentClusterConfigOverrides(), newOverrides); + + return ConfigOverridesHelper.serialize(mergedConfigOverrides); + } + + /** + * Merges the current set of overrides with the new set and returns a new instance + * of the merged config overrides. + * + * @param other the other ConfigOverrides to merge from. + * @return A new instance of the ConfigOverrides representing the merged config + * override. + */ + private ConfigOverrides merge(final ConfigOverrides current, final ConfigOverrides other) { + final ConfigOverrides merged = new ConfigOverrides(); + ConfigOverrides.Overrides optionalCurrentEnabled = Optional.ofNullable(current.getEnable()) + .orElseGet(ConfigOverrides.Overrides::new); + ConfigOverrides.Overrides optionalCurrentDisabled = Optional.ofNullable(current.getDisable()) + .orElseGet(ConfigOverrides.Overrides::new); + ConfigOverrides.Overrides optionalNewEnable = Optional.ofNullable(other.getEnable()) + .orElseGet(ConfigOverrides.Overrides::new); + ConfigOverrides.Overrides optionalNewDisable = Optional.ofNullable(other.getDisable()) + .orElseGet(ConfigOverrides.Overrides::new); + + mergeRcas(merged, optionalCurrentEnabled, optionalNewEnable, optionalCurrentDisabled, optionalNewDisable); + mergeDeciders(merged, optionalCurrentEnabled, optionalNewEnable, optionalCurrentDisabled, optionalNewDisable); + mergeActions(merged, optionalCurrentEnabled, optionalNewEnable, optionalCurrentDisabled, optionalNewDisable); + + return merged; + } + + private void mergeRcas(final ConfigOverrides merged, + final ConfigOverrides.Overrides baseEnabled, + final ConfigOverrides.Overrides newEnabled, + final ConfigOverrides.Overrides baseDisabled, + final ConfigOverrides.Overrides newDisabled) { + List currentRcaEnabled = Optional.ofNullable(baseEnabled.getRcas()) + .orElseGet(ArrayList::new); + List currentRcaDisabled = Optional.ofNullable(baseDisabled.getRcas()) + .orElseGet(ArrayList::new); + List requestedRcasEnabled = Optional.ofNullable(newEnabled.getRcas()) + .orElseGet(ArrayList::new); + List requestedRcasDisabled = Optional.ofNullable(newDisabled.getRcas()) + .orElseGet(ArrayList::new); + + List mergedRcasEnabled = combineLists(currentRcaEnabled, requestedRcasEnabled, requestedRcasDisabled); + List mergedRcasDisabled = combineLists(currentRcaDisabled, requestedRcasDisabled, requestedRcasEnabled); + + merged.getEnable().setRcas(mergedRcasEnabled); + merged.getDisable().setRcas(mergedRcasDisabled); + } + + private void mergeDeciders(final ConfigOverrides merged, + final ConfigOverrides.Overrides baseEnabled, + final ConfigOverrides.Overrides newEnabled, + final ConfigOverrides.Overrides baseDisabled, + final ConfigOverrides.Overrides newDisabled) { + List currentDecidersEnabled = Optional.ofNullable(baseEnabled.getDeciders()) + .orElseGet(ArrayList::new); + List currentDecidersDisabled = Optional.ofNullable(baseDisabled.getDeciders()) + .orElseGet(ArrayList::new); + List requestedDecidersEnabled = Optional.ofNullable(newEnabled.getDeciders()) + .orElseGet(ArrayList::new); + List requestedDecidersDisabled = Optional.ofNullable(newDisabled.getDeciders()) + .orElseGet(ArrayList::new); + + List mergedDecidersEnabled = combineLists(currentDecidersEnabled, requestedDecidersEnabled, requestedDecidersDisabled); + List mergedDecidersDisabled = combineLists(currentDecidersDisabled, requestedDecidersDisabled, requestedDecidersEnabled); + + merged.getEnable().setDeciders(mergedDecidersEnabled); + merged.getDisable().setDeciders(mergedDecidersDisabled); + } + + private void mergeActions(final ConfigOverrides merged, + final ConfigOverrides.Overrides baseEnabled, + final ConfigOverrides.Overrides newEnabled, + final ConfigOverrides.Overrides baseDisabled, + final ConfigOverrides.Overrides newDisabled) { + List currentActionsEnabled = Optional.ofNullable(baseEnabled.getActions()) + .orElseGet(ArrayList::new); + List currentActionsDisabled = Optional.ofNullable(baseDisabled.getActions()) + .orElseGet(ArrayList::new); + List requestedActionsEnabled = Optional.ofNullable(newEnabled.getActions()) + .orElseGet(ArrayList::new); + List requestedActionsDisabled = Optional.ofNullable(newDisabled.getActions()) + .orElseGet(ArrayList::new); + + List mergedActionsEnabled = combineLists(currentActionsEnabled, requestedActionsEnabled, requestedActionsDisabled); + List mergedActionsDisabled = combineLists(currentActionsDisabled, requestedActionsDisabled, requestedActionsEnabled); + + merged.getEnable().setActions(mergedActionsEnabled); + merged.getDisable().setActions(mergedActionsDisabled); + } + + /** + * Combines three lists by adding all elements in the addList to the base list and + * removing all elements in the remove list from the combined list. + * // TODO: Add example here to clarify + * + * @param baseList The base list. + * @param addList The list whose contents need to added to the base list. + * @param removeList The list whose contents should be removed from the base list if present. + * @return The combined list as an immutable list. + */ + private List combineLists(List baseList, List addList, List removeList) { + Set combinedEnabled = new HashSet<>(baseList); + combinedEnabled.addAll(addList); + combinedEnabled.removeAll(removeList); + + return ImmutableList.copyOf(combinedEnabled); + } + +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigAction.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigAction.java new file mode 100644 index 00000000..263ab474 --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigAction.java @@ -0,0 +1,191 @@ +/* + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverrides; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesHelper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler.ConfigOverridesClusterSettingHandler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.Collections; + +/** + * Rest request handler for handling config overrides for various performance analyzer features. + */ +public class PerformanceAnalyzerOverridesClusterConfigAction extends BaseRestHandler { + + private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerOverridesClusterConfigAction.class); + private static final String PA_CONFIG_OVERRIDES_PATH = "/_opendistro/_performanceanalyzer/override/cluster/config"; + private static final String OVERRIDES_FIELD = "overrides"; + private static final String REASON_FIELD = "reason"; + private static final String OVERRIDE_TRIGGERED_FIELD = "override triggered"; + + private final ConfigOverridesClusterSettingHandler configOverridesClusterSettingHandler; + private final ConfigOverridesWrapper overridesWrapper; + + public PerformanceAnalyzerOverridesClusterConfigAction( + final Settings settings, final RestController restController, + final ConfigOverridesClusterSettingHandler configOverridesClusterSettingHandler, + final ConfigOverridesWrapper overridesWrapper) { + super(settings); + this.configOverridesClusterSettingHandler = configOverridesClusterSettingHandler; + this.overridesWrapper = overridesWrapper; + registerHandlers(restController); + } + + private void registerHandlers(final RestController restController) { + restController.registerHandler(RestRequest.Method.GET, PA_CONFIG_OVERRIDES_PATH, this); + restController.registerHandler(RestRequest.Method.POST, PA_CONFIG_OVERRIDES_PATH, this); + } + + /** + * @return the name of this handler. + */ + @Override + public String getName() { + return PerformanceAnalyzerOverridesClusterConfigAction.class.getSimpleName(); + } + + /** + * Prepare the request for execution. Implementations should consume all request params before + * returning the runnable for actual execution. Unconsumed params will immediately terminate + * execution of the request. However, some params are only used in processing the response; + * implementations can override {@link BaseRestHandler#responseParams()} to indicate such params. + * + * @param request the request to execute + * @param client client for executing actions on the local node + * @return the action to execute + * @throws IOException if an I/O exception occurred parsing the request and preparing for + * execution + */ + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) + throws IOException { + RestChannelConsumer consumer; + if (request.method() == RestRequest.Method.GET) { + consumer = handleGet(); + } else if (request.method() == RestRequest.Method.POST) { + consumer = handlePost(request); + } else { + String reason = "Unsupported method:" + request.method().toString() + " Supported: [GET, POST]"; + consumer = sendErrorResponse(reason, RestStatus.METHOD_NOT_ALLOWED); + } + + return consumer; + } + + /** + * Handler for the GET method. + * + * @return RestChannelConsumer that sends the current config overrides when run. + */ + private RestChannelConsumer handleGet() { + return channel -> { + try { + final ConfigOverrides overrides = overridesWrapper.getCurrentClusterConfigOverrides(); + XContentBuilder builder = channel.newBuilder(); + builder.startObject(); + builder.field(OVERRIDES_FIELD, ConfigOverridesHelper.serialize(overrides)); + builder.endObject(); + channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); + } catch (IOException ioe) { + LOG.error("Error sending response", ioe); + } + }; + } + + /** + * Handler for the POST method. + * + * @param request The POST request. + * @return RestChannelConsumer that updates the cluster setting with the requested config + * overrides when run. + * @throws IOException if an exception occurs trying to parse or execute the request. + */ + private RestChannelConsumer handlePost(final RestRequest request) throws IOException { + String jsonString = XContentHelper.convertToJson(request.content(), false, XContentType.JSON); + ConfigOverrides requestedOverrides = ConfigOverridesHelper.deserialize(jsonString); + + if (!validateOverrides(requestedOverrides)) { + String reason = "enable set and disable set should be disjoint"; + return sendErrorResponse(reason, RestStatus.BAD_REQUEST); + } + + configOverridesClusterSettingHandler.updateConfigOverrides(requestedOverrides); + return channel -> { + XContentBuilder builder = channel.newBuilder(); + builder.startObject(); + builder.field(OVERRIDE_TRIGGERED_FIELD, true); + builder.endObject(); + channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); + }; + } + + private boolean validateOverrides(final ConfigOverrides requestedOverrides) { + boolean isValid = true; + + // Check if we have both enable and disable components + if (requestedOverrides.getDisable() == null || requestedOverrides.getEnable() == null) { + return true; + } + + // Check if any RCA nodes are present in both enabled and disabled lists. + if (requestedOverrides.getEnable().getRcas() != null + && requestedOverrides.getDisable().getRcas() != null) { + isValid = Collections.disjoint(requestedOverrides.getEnable().getRcas(), requestedOverrides.getDisable().getRcas()); + } + + // Check if any deciders are present in both enabled and disabled lists. + if (isValid + && requestedOverrides.getEnable().getDeciders() != null + && requestedOverrides.getDisable().getDeciders() != null) { + isValid = Collections.disjoint(requestedOverrides.getEnable().getDeciders(), requestedOverrides.getDisable().getDeciders()); + } + + // Check if any remediation actions are in both enabled and disabled lists. + if (isValid + && requestedOverrides.getEnable().getActions() != null + && requestedOverrides.getDisable().getActions() != null) { + isValid = Collections.disjoint(requestedOverrides.getEnable().getActions(), requestedOverrides.getDisable().getActions()); + } + + return isValid; + } + + private RestChannelConsumer sendErrorResponse(final String reason, final RestStatus status) { + return channel -> { + XContentBuilder errorBuilder = channel.newErrorBuilder(); + errorBuilder.startObject(); + errorBuilder.field(REASON_FIELD, reason); + errorBuilder.endObject(); + + channel.sendResponse(new BytesRestResponse(status, errorBuilder)); + }; + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigOverridesTestHelper.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigOverridesTestHelper.java new file mode 100644 index 00000000..4677464b --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigOverridesTestHelper.java @@ -0,0 +1,57 @@ +/* + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverrides; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.Arrays; +import java.util.List; + +public class ConfigOverridesTestHelper { + private static final ObjectMapper MAPPER = new ObjectMapper(); + public static final String RCA1 = "rca1"; + public static final String RCA2 = "rca2"; + public static final String RCA3 = "rca3"; + public static final String RCA4 = "rca4"; + public static final String ACTION1 = "act1"; + public static final String ACTION2 = "act2"; + public static final String ACTION3 = "act3"; + public static final String ACTION4 = "act4"; + public static final String DECIDER1 = "dec1"; + public static final String DECIDER2 = "dec2"; + public static final String DECIDER3 = "dec3"; + public static final String DECIDER4 = "dec4"; + public static final List DISABLED_RCAS_LIST = Arrays.asList(RCA1, RCA2); + public static final List ENABLED_RCAS_LIST = Arrays.asList(RCA3, RCA4); + public static final List DISABLED_ACTIONS_LIST = Arrays.asList(ACTION1, ACTION2); + public static final List ENABLED_DECIDERS_LIST = Arrays.asList(DECIDER3, DECIDER4); + + public static ConfigOverrides buildValidConfigOverrides() { + ConfigOverrides overrides = new ConfigOverrides(); + overrides.getDisable().setRcas(DISABLED_RCAS_LIST); + overrides.getDisable().setActions(DISABLED_ACTIONS_LIST); + overrides.getEnable().setRcas(ENABLED_RCAS_LIST); + overrides.getEnable().setDeciders(ENABLED_DECIDERS_LIST); + + return overrides; + } + + public static String getValidConfigOverridesJson() throws JsonProcessingException { + return MAPPER.writeValueAsString(buildValidConfigOverrides()); + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerClusterSettingHandlerTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerClusterSettingHandlerTest.java index 7a1ee31e..d8e39ee4 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerClusterSettingHandlerTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerClusterSettingHandlerTest.java @@ -1,5 +1,5 @@ /* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/ConfigOverridesClusterSettingHandlerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/ConfigOverridesClusterSettingHandlerTests.java new file mode 100644 index 00000000..74fdaf59 --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/ConfigOverridesClusterSettingHandlerTests.java @@ -0,0 +1,158 @@ +/* + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverrides; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.ClusterSettingsManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.elasticsearch.common.settings.Setting; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.ACTION1; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.ACTION2; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.DECIDER1; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.DECIDER2; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.DECIDER3; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.DECIDER4; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.RCA1; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.RCA2; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.RCA3; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.ConfigOverridesTestHelper.RCA4; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +public class ConfigOverridesClusterSettingHandlerTests { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String TEST_KEY = "test key"; + private static final ConfigOverrides EMPTY_OVERRIDES = new ConfigOverrides(); + private ConfigOverridesClusterSettingHandler testClusterSettingHandler; + private ConfigOverridesWrapper testOverridesWrapper; + private Setting testSetting; + private ConfigOverrides testOverrides; + + @Mock + private ClusterSettingsManager mockClusterSettingsManager; + + @Captor + private ArgumentCaptor updatedClusterSettingCaptor; + + @Before + public void setUp() { + initMocks(this); + this.testSetting = Setting.simpleString(TEST_KEY); + this.testOverridesWrapper = new ConfigOverridesWrapper(); + this.testOverrides = ConfigOverridesTestHelper.buildValidConfigOverrides(); + testOverridesWrapper.setCurrentClusterConfigOverrides(EMPTY_OVERRIDES); + + this.testClusterSettingHandler = new ConfigOverridesClusterSettingHandler( + testOverridesWrapper, mockClusterSettingsManager, testSetting); + } + + @Test + public void onSettingUpdateSuccessTest() throws JsonProcessingException { + String updatedSettingValue = ConfigOverridesTestHelper.getValidConfigOverridesJson(); + testClusterSettingHandler.onSettingUpdate(updatedSettingValue); + + assertEquals(updatedSettingValue, MAPPER.writeValueAsString(testOverridesWrapper.getCurrentClusterConfigOverrides())); + } + + @Test + public void onSettingUpdateFailureTest() throws IOException { + String updatedSettingValue = "invalid json"; + ConfigOverridesWrapper failingOverridesWrapper = new ConfigOverridesWrapper(); + + testClusterSettingHandler = new ConfigOverridesClusterSettingHandler( + failingOverridesWrapper, mockClusterSettingsManager, testSetting); + + testClusterSettingHandler.onSettingUpdate(updatedSettingValue); + + assertEquals(MAPPER.writeValueAsString(EMPTY_OVERRIDES), + MAPPER.writeValueAsString(testOverridesWrapper.getCurrentClusterConfigOverrides())); + } + + @Test + public void updateConfigOverridesMergeSuccessTest() throws IOException { + testOverridesWrapper.setCurrentClusterConfigOverrides(testOverrides); + + ConfigOverrides expectedOverrides = new ConfigOverrides(); + ConfigOverrides additionalOverrides = new ConfigOverrides(); + // current enabled rcas: 3,4. current disabled rcas: 1,2 + additionalOverrides.getEnable().setRcas(Arrays.asList(RCA1, RCA1)); + + expectedOverrides.getEnable().setRcas(Arrays.asList(RCA1, RCA3, RCA4)); + expectedOverrides.getDisable().setRcas(Collections.singletonList(RCA2)); + + // current enabled deciders: 3,4. current disabled deciders: none + additionalOverrides.getDisable().setDeciders(Arrays.asList(DECIDER3, DECIDER1)); + additionalOverrides.getEnable().setDeciders(Collections.singletonList(DECIDER2)); + + expectedOverrides.getEnable().setDeciders(Arrays.asList(DECIDER2, DECIDER4)); + expectedOverrides.getDisable().setDeciders(Arrays.asList(DECIDER3, DECIDER1)); + + // current enabled actions: none. current disabled actions: 1,2 + additionalOverrides.getEnable().setActions(Arrays.asList(ACTION1, ACTION2)); + + expectedOverrides.getEnable().setActions(Arrays.asList(ACTION1, ACTION2)); + + testClusterSettingHandler.updateConfigOverrides(additionalOverrides); + verify(mockClusterSettingsManager).updateSetting(eq(testSetting), updatedClusterSettingCaptor.capture()); + + assertTrue(areEqual(expectedOverrides, MAPPER.readValue(updatedClusterSettingCaptor.getValue(), ConfigOverrides.class))); + } + + private boolean areEqual(final ConfigOverrides expected, final ConfigOverrides actual) { + Collections.sort(expected.getEnable().getRcas()); + Collections.sort(actual.getEnable().getRcas()); + assertEquals(expected.getEnable().getRcas(), actual.getEnable().getRcas()); + + Collections.sort(expected.getEnable().getActions()); + Collections.sort(actual.getEnable().getActions()); + assertEquals(expected.getEnable().getActions(), actual.getEnable().getActions()); + + Collections.sort(expected.getEnable().getDeciders()); + Collections.sort(actual.getEnable().getDeciders()); + assertEquals(expected.getEnable().getDeciders(), actual.getEnable().getDeciders()); + + Collections.sort(expected.getDisable().getRcas()); + Collections.sort(actual.getDisable().getRcas()); + assertEquals(expected.getDisable().getRcas(), actual.getDisable().getRcas()); + + Collections.sort(expected.getDisable().getActions()); + Collections.sort(actual.getDisable().getActions()); + assertEquals(expected.getDisable().getActions(), actual.getDisable().getActions()); + + Collections.sort(expected.getDisable().getDeciders()); + Collections.sort(actual.getDisable().getDeciders()); + assertEquals(expected.getDisable().getDeciders(), actual.getDisable().getDeciders()); + + return true; + } +} \ No newline at end of file