From f652c78405c2b9d05e1a256e82c85d68689ce5e6 Mon Sep 17 00:00:00 2001 From: Kaituo Li Date: Mon, 4 May 2020 14:31:32 -0700 Subject: [PATCH] Add settings to disable/enable AD dynamically This PR adds settings to disable/enable AD dynamically. AD plugin rejects RESTful requests and stops AD jobs when it is disabled. Testing done: - added tests to verify if AD plugin rejects RESTful requests when it is disabled - manually verified AD jobs are stopped when AD plugin is disabled --- build.gradle | 5 +- .../ad/AnomalyDetectorPlugin.java | 15 +- .../ad/constant/CommonErrorMessages.java | 3 +- .../ad/rest/AbstractSearchAction.java | 6 + .../ad/rest/RestAnomalyDetectorJobAction.java | 7 + .../rest/RestDeleteAnomalyDetectorAction.java | 9 +- .../RestExecuteAnomalyDetectorAction.java | 5 + .../ad/rest/RestGetAnomalyDetectorAction.java | 5 + .../rest/RestIndexAnomalyDetectorAction.java | 7 + .../rest/RestStatsAnomalyDetectorAction.java | 5 + .../ad/settings/EnabledSetting.java | 111 ++++++++++ .../AnomalyResultTransportAction.java | 8 +- .../ad/AnomalyDetectorRestTestCase.java | 29 +++ .../ad/e2e/DetectionResultEvalutationIT.java | 2 - .../ad/rest/AnomalyDetectorRestApiIT.java | 193 +++++++++++++++++- 15 files changed, 391 insertions(+), 19 deletions(-) create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/EnabledSetting.java diff --git a/build.gradle b/build.gradle index 68607e52..4a2203bd 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -251,6 +251,9 @@ List jacocoExclusions = [ // Class containing just constants. Don't need to test 'com.amazon.opendistroforelasticsearch.ad.constant.*', + // mostly skeleton code. Tested major logic in restful api tests + 'com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting', + 'com.amazon.opendistroforelasticsearch.ad.common.exception.FeatureNotAvailableException', 'com.amazon.opendistroforelasticsearch.ad.common.exception.AnomalyDetectionException', 'com.amazon.opendistroforelasticsearch.ad.util.ClientUtil', diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java index 8cc71423..2e00b360 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java @@ -15,6 +15,8 @@ package com.amazon.opendistroforelasticsearch.ad; +import static java.util.Collections.unmodifiableList; + import com.amazon.opendistroforelasticsearch.ad.breaker.ADCircuitBreakerService; import com.amazon.opendistroforelasticsearch.ad.cluster.ADClusterEventListener; import com.amazon.opendistroforelasticsearch.ad.cluster.ADMetaData; @@ -46,6 +48,7 @@ import com.amazon.opendistroforelasticsearch.ad.rest.RestSearchAnomalyResultAction; import com.amazon.opendistroforelasticsearch.ad.rest.RestStatsAnomalyDetectorAction; import com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.amazon.opendistroforelasticsearch.ad.stats.ADStat; import com.amazon.opendistroforelasticsearch.ad.stats.ADStats; import com.amazon.opendistroforelasticsearch.ad.stats.StatNames; @@ -125,8 +128,8 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; - -import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.AD_THEAD_POOL_QUEUE_SIZE; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Entry point of AD plugin. @@ -245,6 +248,7 @@ public Collection createComponents( NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry ) { + EnabledSetting.getInstance().init(clusterService); this.client = client; this.threadPool = threadPool; Settings settings = environment.settings(); @@ -378,7 +382,7 @@ public List> getExecutorBuilders(Settings settings) { settings, AD_THREAD_POOL_NAME, Math.max(1, EsExecutors.numberOfProcessors(settings) / 4), - AD_THEAD_POOL_QUEUE_SIZE, + AnomalyDetectorSettings.AD_THEAD_POOL_QUEUE_SIZE, "opendistro.ad." + AD_THREAD_POOL_NAME ) ); @@ -386,7 +390,9 @@ public List> getExecutorBuilders(Settings settings) { @Override public List> getSettings() { - return ImmutableList + List> enabledSetting = EnabledSetting.getInstance().getSettings(); + + List> systemSetting = ImmutableList .of( AnomalyDetectorSettings.MAX_ANOMALY_DETECTORS, AnomalyDetectorSettings.MAX_ANOMALY_FEATURES, @@ -403,6 +409,7 @@ public List> getSettings() { AnomalyDetectorSettings.BACKOFF_INITIAL_DELAY, AnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF ); + return unmodifiableList(Stream.concat(enabledSetting.stream(), systemSetting.stream()).collect(Collectors.toList())); } @Override diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java index ec505927..bc9c040e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -24,4 +24,5 @@ public class CommonErrorMessages { public static final String MEMORY_LIMIT_EXCEEDED_ERR_MSG = "AD models memory usage exceeds our limit."; public static final String FEATURE_NOT_AVAILABLE_ERR_MSG = "No Feature in current detection window."; public static final String MEMORY_CIRCUIT_BROKEN_ERR_MSG = "AD memory circuit is broken."; + public static final String DISABLED_ERR_MSG = "AD plugin is disabled. To enable update opendistro.anomaly_detection.enabled to true"; } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/AbstractSearchAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/AbstractSearchAction.java index 3f98befd..46f53b95 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/AbstractSearchAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/AbstractSearchAction.java @@ -15,7 +15,10 @@ package com.amazon.opendistroforelasticsearch.ad.rest; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchRequest; @@ -64,6 +67,9 @@ public AbstractSearchAction(RestController controller, String urlPath, String in @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (!EnabledSetting.isADPluginEnabled()) { + throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG); + } SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()); searchSourceBuilder.fetchSource(getSourceContext(request)); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestAnomalyDetectorJobAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestAnomalyDetectorJobAction.java index cd702781..81a8f39f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestAnomalyDetectorJobAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestAnomalyDetectorJobAction.java @@ -16,8 +16,11 @@ package com.amazon.opendistroforelasticsearch.ad.rest; import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; import com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices; import com.amazon.opendistroforelasticsearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; + import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.service.ClusterService; @@ -86,6 +89,10 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (!EnabledSetting.isADPluginEnabled()) { + throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG); + } + String detectorId = request.param(DETECTOR_ID); return channel -> { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestDeleteAnomalyDetectorAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestDeleteAnomalyDetectorAction.java index dc1a75ca..4263771f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestDeleteAnomalyDetectorAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestDeleteAnomalyDetectorAction.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -18,7 +18,10 @@ import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.rest.handler.AnomalyDetectorActionHandler; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; @@ -69,6 +72,10 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (!EnabledSetting.isADPluginEnabled()) { + throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG); + } + String detectorId = request.param(DETECTOR_ID); return channel -> { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestExecuteAnomalyDetectorAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestExecuteAnomalyDetectorAction.java index 0707948f..0c7cf499 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestExecuteAnomalyDetectorAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestExecuteAnomalyDetectorAction.java @@ -17,8 +17,10 @@ import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorRunner; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorExecutionInput; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyResultAction; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyResultRequest; import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; @@ -106,6 +108,9 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (!EnabledSetting.isADPluginEnabled()) { + throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG); + } AnomalyDetectorExecutionInput input = getAnomalyDetectorExecutionInput(request); return channel -> { String rawPath = request.rawPath(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java index 21e142e6..b52b6966 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java @@ -19,10 +19,12 @@ import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfile; import com.amazon.opendistroforelasticsearch.ad.model.ProfileName; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; import com.google.common.collect.Sets; import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorProfileRunner; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -105,6 +107,9 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (!EnabledSetting.isADPluginEnabled()) { + throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG); + } String detectorId = request.param(DETECTOR_ID); boolean returnJob = request.paramAsBoolean("job", false); String typesStr = request.param(TYPE); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestIndexAnomalyDetectorAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestIndexAnomalyDetectorAction.java index e938fef5..5b98b3d0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestIndexAnomalyDetectorAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestIndexAnomalyDetectorAction.java @@ -18,7 +18,10 @@ import com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.rest.handler.IndexAnomalyDetectorActionHandler; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.support.WriteRequest; @@ -92,6 +95,10 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (!EnabledSetting.isADPluginEnabled()) { + throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG); + } + String detectorId = request.param(DETECTOR_ID, AnomalyDetector.NO_ID); logger.info("AnomalyDetector {} action for detectorId {}", request.method(), detectorId); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestStatsAnomalyDetectorAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestStatsAnomalyDetectorAction.java index 291d0830..a0cd1ddc 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestStatsAnomalyDetectorAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestStatsAnomalyDetectorAction.java @@ -15,6 +15,8 @@ package com.amazon.opendistroforelasticsearch.ad.rest; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.amazon.opendistroforelasticsearch.ad.stats.ADStats; import com.amazon.opendistroforelasticsearch.ad.transport.ADStatsAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADStatsRequest; @@ -67,6 +69,9 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + if (!EnabledSetting.isADPluginEnabled()) { + throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG); + } ADStatsRequest adStatsRequest = getRequest(request); return channel -> client.execute(ADStatsAction.INSTANCE, adStatsRequest, new RestActions.NodesResponseRestListener<>(channel)); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/EnabledSetting.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/EnabledSetting.java new file mode 100644 index 00000000..7b3be00d --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/EnabledSetting.java @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.settings; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.common.settings.Setting.Property.Dynamic; +import static org.elasticsearch.common.settings.Setting.Property.NodeScope; + +public class EnabledSetting { + + private static Logger logger = LogManager.getLogger(EnabledSetting.class); + + /** + * Singleton instance + */ + private static EnabledSetting INSTANCE; + + /** + * Settings name + */ + public static final String AD_PLUGIN_ENABLED = "opendistro.anomaly_detection.enabled"; + + private final Map> settings = unmodifiableMap(new HashMap>() { + { + /** + * AD plugin enable/disable setting + */ + put(AD_PLUGIN_ENABLED, Setting.boolSetting(AD_PLUGIN_ENABLED, true, NodeScope, Dynamic)); + } + }); + + /** Latest setting value for each registered key. Thread-safe is required. */ + private final Map latestSettings = new ConcurrentHashMap<>(); + + private ClusterService clusterService; + + private EnabledSetting() {} + + public static synchronized EnabledSetting getInstance() { + if (INSTANCE == null) { + INSTANCE = new EnabledSetting(); + } + return INSTANCE; + } + + private void setSettingsUpdateConsumers() { + for (Setting setting : settings.values()) { + clusterService.getClusterSettings().addSettingsUpdateConsumer(setting, newVal -> { + logger.info("[AD] The value of setting [{}] changed to [{}]", setting.getKey(), newVal); + latestSettings.put(setting.getKey(), newVal); + }); + } + } + + /** + * Get setting value by key. Return default value if not configured explicitly. + * + * @param key setting key. + * @param Setting type + * @return T setting value or default + */ + @SuppressWarnings("unchecked") + public T getSettingValue(String key) { + return (T) latestSettings.getOrDefault(key, getSetting(key).getDefault(Settings.EMPTY)); + } + + private Setting getSetting(String key) { + if (settings.containsKey(key)) { + return settings.get(key); + } + throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); + } + + public static boolean isADPluginEnabled() { + return EnabledSetting.getInstance().getSettingValue(EnabledSetting.AD_PLUGIN_ENABLED); + } + + public void init(ClusterService clusterService) { + this.clusterService = clusterService; + setSettingsUpdateConsumers(); + } + + public List> getSettings() { + return new ArrayList<>(settings.values()); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyResultTransportAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyResultTransportAction.java index 95c2ba2a..5a779a21 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyResultTransportAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyResultTransportAction.java @@ -42,6 +42,7 @@ import com.amazon.opendistroforelasticsearch.ad.model.FeatureData; import com.amazon.opendistroforelasticsearch.ad.model.IntervalTimeConfiguration; import com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.amazon.opendistroforelasticsearch.ad.stats.ADStats; import com.amazon.opendistroforelasticsearch.ad.stats.StatNames; import com.amazon.opendistroforelasticsearch.ad.util.ColdStartRunner; @@ -183,7 +184,7 @@ private List getFeatureData(double[] currentFeature, AnomalyDetecto * * index does not exist * * all features have been disabled * + anomaly detector is not available - + * + AD plugin is disabled * * Known cause of InternalFailure: * + threshold model node is not available @@ -197,6 +198,7 @@ private List getFeatureData(double[] currentFeature, AnomalyDetecto */ @Override protected void doExecute(Task task, ActionRequest actionRequest, ActionListener listener) { + AnomalyResultRequest request = AnomalyResultRequest.fromActionRequest(actionRequest); ActionListener original = listener; listener = ActionListener.wrap(original::onResponse, e -> { @@ -206,6 +208,10 @@ protected void doExecute(Task task, ActionRequest actionRequest, ActionListener< String adID = request.getAdID(); + if (!EnabledSetting.isADPluginEnabled()) { + throw new EndRunException(adID, CommonErrorMessages.DISABLED_ERR_MSG, true); + } + adStats.getStat(StatNames.AD_EXECUTE_REQUEST_COUNT.getName()).increment(); if (adCircuitBreakerService.isOpen()) { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java index 2d71c522..bdea7b5e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java @@ -17,13 +17,16 @@ import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; +import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -196,4 +199,30 @@ protected RestStatus restStatus(Response response) { protected final XContentParser createAdParser(XContent xContent, InputStream data) throws IOException { return xContent.createParser(TestHelpers.xContentRegistry(), LoggingDeprecationHandler.INSTANCE, data); } + + public void updateClusterSettings(String settingKey, Object value) throws Exception { + XContentBuilder builder = XContentFactory + .jsonBuilder() + .startObject() + .startObject("persistent") + .field(settingKey, value) + .endObject() + .endObject(); + Request request = new Request("PUT", "_cluster/settings"); + request.setJsonEntity(Strings.toString(builder)); + Response response = client().performRequest(request); + assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + + public Response getDetectorProfile(String detectorId) throws IOException { + return TestHelpers + .makeRequest( + client(), + "GET", + TestHelpers.AD_BASE_DETECTORS_URI + "/" + detectorId + "/" + RestHandlerUtils.PROFILE, + null, + "", + ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana")) + ); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/e2e/DetectionResultEvalutationIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/e2e/DetectionResultEvalutationIT.java index d3d5ffeb..b16a9f86 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/e2e/DetectionResultEvalutationIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/e2e/DetectionResultEvalutationIT.java @@ -35,8 +35,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.test.rest.ESRestTestCase; -import static org.junit.Assert.assertTrue; - public class DetectionResultEvalutationIT extends ESRestTestCase { public void testDataset() throws Exception { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java index 753611cc..c6544b98 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -18,10 +18,12 @@ import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorRestTestCase; import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorExecutionInput; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyResult; +import com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.http.entity.ContentType; @@ -40,6 +42,7 @@ import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.AD_BASE_PREVIEW_URI; import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.randomAnomalyDetectorWithEmptyFeature; +import static org.hamcrest.Matchers.containsString; public class AnomalyDetectorRestApiIT extends AnomalyDetectorRestTestCase { @@ -75,11 +78,21 @@ public void testCreateAnomalyDetectorWithEmptyIndices() throws Exception { ); } - public void testCreateAnomalyDetector() throws IOException { + public void testCreateAnomalyDetector() throws Exception { AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TestHelpers.randomUiMetadata(), null); String indexName = detector.getIndices().get(0); TestHelpers.createIndex(client(), indexName, toHttpEntity("{\"name\": \"test\"}")); + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest(client(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), toHttpEntity(detector), null) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); Response response = TestHelpers .makeRequest(client(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), toHttpEntity(detector), null); assertEquals("Create anomaly detector failed", RestStatus.CREATED, restStatus(response)); @@ -90,8 +103,16 @@ public void testCreateAnomalyDetector() throws IOException { assertTrue("incorrect version", version > 0); } - public void testGetAnomalyDetector() throws IOException { + public void testGetAnomalyDetector() throws Exception { AnomalyDetector detector = createRandomAnomalyDetector(true, true); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows(ResponseException.class, () -> getAnomalyDetector(detector.getDetectorId())); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + AnomalyDetector createdDetector = getAnomalyDetector(detector.getDetectorId()); assertEquals("Incorrect Location header", detector, createdDetector); } @@ -101,7 +122,7 @@ public void testGetNotExistingAnomalyDetector() throws Exception { TestHelpers.assertFailWith(ResponseException.class, null, () -> getAnomalyDetector(randomAlphaOfLength(5))); } - public void testUpdateAnomalyDetectorA() throws IOException { + public void testUpdateAnomalyDetectorA() throws Exception { AnomalyDetector detector = createRandomAnomalyDetector(true, true); String newDescription = randomAlphaOfLength(5); @@ -122,6 +143,24 @@ public void testUpdateAnomalyDetectorA() throws IOException { detector.getLastUpdateTime() ); + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "PUT", + TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "?refresh=true", + ImmutableMap.of(), + toHttpEntity(newDetector), + null + ) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response updateResponse = TestHelpers .makeRequest( client(), @@ -181,9 +220,28 @@ public void testUpdateAnomalyDetectorWithNotExistingIndex() throws Exception { ); } - public void testSearchAnomalyDetector() throws IOException { + public void testSearchAnomalyDetector() throws Exception { AnomalyDetector detector = createRandomAnomalyDetector(true, true); SearchSourceBuilder search = (new SearchSourceBuilder()).query(QueryBuilders.termQuery("_id", detector.getDetectorId())); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "GET", + TestHelpers.AD_BASE_DETECTORS_URI + "/_search", + ImmutableMap.of(), + new NStringEntity(search.toString(), ContentType.APPLICATION_JSON), + null + ) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response searchResponse = TestHelpers .makeRequest( client(), @@ -196,14 +254,23 @@ public void testSearchAnomalyDetector() throws IOException { assertEquals("Search anomaly detector failed", RestStatus.OK, restStatus(searchResponse)); } - public void testStatsAnomalyDetector() throws IOException { + public void testStatsAnomalyDetector() throws Exception { + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers.makeRequest(client(), "GET", AnomalyDetectorPlugin.AD_BASE_URI + "/stats", ImmutableMap.of(), "", null) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response statsResponse = TestHelpers .makeRequest(client(), "GET", AnomalyDetectorPlugin.AD_BASE_URI + "/stats", ImmutableMap.of(), "", null); assertEquals("Get stats failed", RestStatus.OK, restStatus(statsResponse)); } - public void testPreviewAnomalyDetector() throws IOException { + public void testPreviewAnomalyDetector() throws Exception { AnomalyDetector detector = createRandomAnomalyDetector(true, false); AnomalyDetectorExecutionInput input = new AnomalyDetectorExecutionInput( detector.getDetectorId(), @@ -211,6 +278,25 @@ public void testPreviewAnomalyDetector() throws IOException { Instant.now(), null ); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "POST", + String.format(AD_BASE_PREVIEW_URI, input.getDetectorId()), + ImmutableMap.of(), + toHttpEntity(input), + null + ) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response response = TestHelpers .makeRequest( client(), @@ -312,7 +398,7 @@ public void testPreviewAnomalyDetectorWithDetectorAndNoFeatures() throws Excepti ); } - public void testSearchAnomalyResult() throws IOException { + public void testSearchAnomalyResult() throws Exception { AnomalyResult anomalyResult = TestHelpers.randomAnomalyDetectResult(); Response response = TestHelpers .makeRequest( @@ -327,6 +413,25 @@ public void testSearchAnomalyResult() throws IOException { SearchSourceBuilder search = (new SearchSourceBuilder()) .query(QueryBuilders.termQuery("detector_id", anomalyResult.getDetectorId())); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "POST", + TestHelpers.AD_BASE_RESULT_URI + "/_search", + ImmutableMap.of(), + new NStringEntity(search.toString(), ContentType.APPLICATION_JSON), + null + ) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response searchResponse = TestHelpers .makeRequest( client(), @@ -351,8 +456,27 @@ public void testSearchAnomalyResult() throws IOException { assertEquals("Search anomaly result failed", RestStatus.OK, restStatus(searchAllResponse)); } - public void testDeleteAnomalyDetector() throws IOException { + public void testDeleteAnomalyDetector() throws Exception { AnomalyDetector detector = createRandomAnomalyDetector(true, false); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "DELETE", + TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId(), + ImmutableMap.of(), + "", + null + ) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response response = TestHelpers .makeRequest( client(), @@ -503,6 +627,24 @@ public void testGetDetectorWithAdJob() throws IOException { public void testStartAdJobWithExistingDetector() throws Exception { AnomalyDetector detector = createRandomAnomalyDetector(true, false); + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "POST", + TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", + ImmutableMap.of(), + "", + null + ) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response startAdJobResponse = TestHelpers .makeRequest( client(), @@ -564,6 +706,7 @@ public void testStartAdJobWithNonexistingDetector() throws Exception { } public void testStopAdJob() throws Exception { + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); AnomalyDetector detector = createRandomAnomalyDetector(true, false); Response startAdJobResponse = TestHelpers .makeRequest( @@ -576,6 +719,24 @@ public void testStopAdJob() throws Exception { ); assertEquals("Fail to start AD job", RestStatus.OK, restStatus(startAdJobResponse)); + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "POST", + TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_stop", + ImmutableMap.of(), + "", + null + ) + ); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + Response stopAdJobResponse = TestHelpers .makeRequest( client(), @@ -723,4 +884,18 @@ public void testStartAdjobWithEmptyFeatures() throws Exception { ) ); } + + public void testProfileAnomalyDetector() throws Exception { + AnomalyDetector detector = createRandomAnomalyDetector(true, true); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); + + Exception ex = expectThrows(ResponseException.class, () -> getDetectorProfile(detector.getDetectorId())); + assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + + Response profileResponse = getDetectorProfile(detector.getDetectorId()); + assertEquals("Incorrect profile status", RestStatus.OK, restStatus(profileResponse)); + } }