diff --git a/README.md b/README.md index 376d3a83b..ba09eacdd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![AD Test](https://github.com/opensearch-project/anomaly-detection/workflows/Build%20and%20Test%20Anomaly%20detection/badge.svg)](https://github.com/opensearch-project/anomaly-detection/actions?query=workflow%3A%22Build+and+Test+Anomaly+detection%22+branch%3A%22main%22) [![codecov](https://codecov.io/gh/opensearch-project/anomaly-detection/branch/main/graph/badge.svg?flag=plugin)](https://codecov.io/gh/opensearch-project/anomaly-detection) -[![Documentation](https://img.shields.io/badge/doc-reference-blue)](https://opendistro.github.io/for-elasticsearch-docs/docs/ad/) +[![Documentation](https://img.shields.io/badge/doc-reference-blue)](https://docs-beta.opensearch.org/docs/ad/) [![Forum](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/Use-this-category-for-all-questions-around-machine-learning-plugins) ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) @@ -31,7 +31,7 @@ You should use anomaly detection plugin with the same version of [OpenSearch Ale ## Documentation -Please see [our documentation](https://opendistro.github.io/for-elasticsearch-docs/docs/ad/). +Please see [our documentation](https://docs-beta.opensearch.org/docs/ad/). ## Contributing diff --git a/build.gradle b/build.gradle index 6de225401..829421855 100644 --- a/build.gradle +++ b/build.gradle @@ -278,7 +278,7 @@ evaluationDependsOnChildren() task release(type: Copy, group: 'build') { dependsOn allprojects*.tasks.build from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile())) - into "build/plugins/opendistro-anomaly-detection" + into "build/plugins/opensearch-anomaly-detection" includeEmptyDirs = false // ES versions < 6.3 have a top-level opensearch directory inside the plugin zip which we need to remove eachFile { it.path = it.path - "opensearch/" } @@ -327,6 +327,23 @@ List jacocoExclusions = [ 'org.opensearch.ad.indices.AnomalyDetectionIndices', 'org.opensearch.ad.transport.handler.MultiEntityResultHandler', 'org.opensearch.ad.util.ThrowingSupplierWrapper', + 'org.opensearch.ad.ratelimit.*', + 'org.opensearch.ad.transport.EntityResultTransportAction', + 'org.opensearch.ad.transport.EntityResultTransportAction.*', + 'org.opensearch.ad.transport.AnomalyResultTransportAction.*', + 'org.opensearch.ad.transport.ProfileNodeResponse', + 'org.opensearch.ad.transport.ADResultBulkResponse', + 'org.opensearch.ad.transport.AggregationType', + 'org.opensearch.ad.EntityProfileRunner', + 'org.opensearch.ad.NodeStateManager', + 'org.opensearch.ad.util.BulkUtil', + 'org.opensearch.ad.util.ExceptionUtil', + 'org.opensearch.ad.feature.SearchFeatureDao', + 'org.opensearch.ad.feature.CompositeRetriever.*', + 'org.opensearch.ad.feature.ScriptMaker', + 'org.opensearch.ad.ml.EntityModel', + 'org.opensearch.ad.caching.PriorityCache', + 'org.opensearch.ad.ml.CheckpointDao', ] jacocoTestCoverageVerification { @@ -431,11 +448,11 @@ afterEvaluate { prefix '/usr' license 'ASL-2.0' - maintainer 'OpenDistro for Elasticsearch Team ' - url 'https://opendistro.github.io/for-elasticsearch/downloads.html' + maintainer 'OpenSearch ' + url 'https://opensearch.org/downloads.html' summary ''' - Anomaly Detection plugin for OpenDistro for Elasticsearch. - Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/. + Anomaly Detection plugin for OpenSearch. + Reference documentation can be found at https://docs-beta.opensearch.org/docs/ad/. '''.stripIndent().replace('\n', ' ').trim() } diff --git a/src/main/java/org/opensearch/ad/constant/CommonMessageAttributes.java b/src/main/java/org/opensearch/ad/constant/CommonMessageAttributes.java deleted file mode 100644 index ed8840d6f..000000000 --- a/src/main/java/org/opensearch/ad/constant/CommonMessageAttributes.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package org.opensearch.ad.constant; - -public class CommonMessageAttributes { - - // ====================================== - // Json keys - // ====================================== - public static final String RCF_SCORE_JSON_KEY = "rCFScore"; - public static final String ID_JSON_KEY = "adID"; - public static final String MODEL_ID_JSON_KEY = "modelID"; - public static final String FEATURE_JSON_KEY = "features"; - public static final String CONFIDENCE_JSON_KEY = "confidence"; - public static final String ANOMALY_GRADE_JSON_KEY = "anomalyGrade"; - public static final String QUEUE_JSON_KEY = "queue"; - public static final String START_JSON_KEY = "start"; - public static final String END_JSON_KEY = "end"; -} diff --git a/src/main/java/org/opensearch/ad/constant/CommonName.java b/src/main/java/org/opensearch/ad/constant/CommonName.java index e29e56e76..2a3721445 100644 --- a/src/main/java/org/opensearch/ad/constant/CommonName.java +++ b/src/main/java/org/opensearch/ad/constant/CommonName.java @@ -73,11 +73,17 @@ public class CommonName { public static final String MODELS = "models"; public static final String MODEL = "model"; public static final String INIT_PROGRESS = "init_progress"; + public static final String MODEL_SIZE_IN_BYTES = "model_size_in_bytes"; + public static final String CATEGORICAL_FIELD = "category_field"; public static final String TOTAL_ENTITIES = "total_entities"; public static final String ACTIVE_ENTITIES = "active_entities"; public static final String ENTITY_INFO = "entity_info"; public static final String TOTAL_UPDATES = "total_updates"; + + // ====================================== + // Historical detectors + // ====================================== public static final String AD_TASK = "ad_task"; public static final String AD_TASK_REMOTE = "ad_task_remote"; public static final String CANCEL_TASK = "cancel_task"; @@ -107,4 +113,25 @@ public class CommonName { public static final String DATE_HISTOGRAM = "date_histogram"; // feature aggregation name public static final String FEATURE_AGGS = "feature_aggs"; + + // ====================================== + // Used in almost all components + // ====================================== + public static final String MODEL_ID_KEY = "model_id"; + public static final String DETECTOR_ID_KEY = "detector_id"; + public static final String ENTITY_KEY = "entity"; + + // ====================================== + // Used in toXContent + // ====================================== + public static final String RCF_SCORE_JSON_KEY = "rCFScore"; + public static final String ID_JSON_KEY = "adID"; + public static final String FEATURE_JSON_KEY = "features"; + public static final String CONFIDENCE_JSON_KEY = "confidence"; + public static final String ANOMALY_GRADE_JSON_KEY = "anomalyGrade"; + public static final String QUEUE_JSON_KEY = "queue"; + public static final String START_JSON_KEY = "start"; + public static final String END_JSON_KEY = "end"; + public static final String VALUE_JSON_KEY = "value"; + public static final String ENTITIES_JSON_KEY = "entities"; } diff --git a/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java b/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java index 0e255c710..27a338bb8 100644 --- a/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.CommonErrorMessages; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.InputStreamStreamInput; import org.opensearch.common.io.stream.OutputStreamStreamOutput; @@ -103,9 +103,9 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CommonMessageAttributes.ID_JSON_KEY, adID); - builder.field(CommonMessageAttributes.START_JSON_KEY, start); - builder.field(CommonMessageAttributes.END_JSON_KEY, end); + builder.field(CommonName.ID_JSON_KEY, adID); + builder.field(CommonName.START_JSON_KEY, start); + builder.field(CommonName.END_JSON_KEY, end); builder.endObject(); return builder; } diff --git a/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java b/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java index a7915a962..3fd827f89 100644 --- a/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java +++ b/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.ad.constant.CommonErrorMessages; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; @@ -84,7 +84,7 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CommonMessageAttributes.ID_JSON_KEY, adID); + builder.field(CommonName.ID_JSON_KEY, adID); builder.endObject(); return builder; } diff --git a/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java b/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java index 92de00a3d..25ccfa0fc 100644 --- a/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java +++ b/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.CommonErrorMessages; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -75,7 +75,7 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CommonMessageAttributes.ID_JSON_KEY, adID); + builder.field(CommonName.ID_JSON_KEY, adID); builder.endObject(); return builder; } diff --git a/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java b/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java index 8c63d0bb3..4a88c3cfe 100644 --- a/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.CommonErrorMessages; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -107,9 +107,9 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CommonMessageAttributes.ID_JSON_KEY, adID); - builder.field(CommonMessageAttributes.MODEL_ID_JSON_KEY, modelID); - builder.startArray(CommonMessageAttributes.FEATURE_JSON_KEY); + builder.field(CommonName.ID_JSON_KEY, adID); + builder.field(CommonName.MODEL_ID_KEY, modelID); + builder.startArray(CommonName.FEATURE_JSON_KEY); for (double feature : features) { builder.value(feature); } diff --git a/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java b/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java index c02da2ef6..22ff7e23b 100644 --- a/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java +++ b/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.CommonErrorMessages; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.InputStreamStreamInput; import org.opensearch.common.io.stream.OutputStreamStreamOutput; @@ -87,7 +87,7 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CommonMessageAttributes.ID_JSON_KEY, adID); + builder.field(CommonName.ID_JSON_KEY, adID); builder.endObject(); return builder; } diff --git a/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java b/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java index b824475b1..d020b226c 100644 --- a/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.CommonErrorMessages; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -99,9 +99,9 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CommonMessageAttributes.ID_JSON_KEY, adID); - builder.field(CommonMessageAttributes.MODEL_ID_JSON_KEY, modelID); - builder.field(CommonMessageAttributes.RCF_SCORE_JSON_KEY, rcfScore); + builder.field(CommonName.ID_JSON_KEY, adID); + builder.field(CommonName.MODEL_ID_KEY, modelID); + builder.field(CommonName.RCF_SCORE_JSON_KEY, rcfScore); builder.endObject(); return builder; } diff --git a/src/main/java/org/opensearch/ad/transport/ThresholdResultResponse.java b/src/main/java/org/opensearch/ad/transport/ThresholdResultResponse.java index e877c0fa3..e59f37989 100644 --- a/src/main/java/org/opensearch/ad/transport/ThresholdResultResponse.java +++ b/src/main/java/org/opensearch/ad/transport/ThresholdResultResponse.java @@ -29,7 +29,7 @@ import java.io.IOException; import org.opensearch.action.ActionResponse; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.ToXContentObject; @@ -67,8 +67,8 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CommonMessageAttributes.ANOMALY_GRADE_JSON_KEY, anomalyGrade); - builder.field(CommonMessageAttributes.CONFIDENCE_JSON_KEY, confidence); + builder.field(CommonName.ANOMALY_GRADE_JSON_KEY, anomalyGrade); + builder.field(CommonName.CONFIDENCE_JSON_KEY, confidence); builder.endObject(); return builder; } diff --git a/src/test/java/org/opensearch/ad/AbstractADTest.java b/src/test/java/org/opensearch/ad/AbstractADTest.java index 22fe58dc9..7b8b86501 100644 --- a/src/test/java/org/opensearch/ad/AbstractADTest.java +++ b/src/test/java/org/opensearch/ad/AbstractADTest.java @@ -36,8 +36,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; @@ -57,9 +59,11 @@ import org.opensearch.ad.model.AnomalyDetectorJob; import org.opensearch.ad.model.AnomalyResult; import org.opensearch.ad.model.DetectorInternalState; +import org.opensearch.ad.model.Entity; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.http.HttpRequest; @@ -212,7 +216,7 @@ protected static void setUpThreadPool(String name) { AnomalyDetectorPlugin.AD_THREAD_POOL_NAME, 1, 1000, - "opendistro.ad." + AnomalyDetectorPlugin.AD_THREAD_POOL_NAME + "opensearch.ad." + AnomalyDetectorPlugin.AD_THREAD_POOL_NAME ) ); } @@ -223,17 +227,33 @@ protected static void tearDownThreadPool() { threadPool = null; } - public void setupTestNodes(Settings settings, TransportInterceptor transportInterceptor) { + /** + * + * @param transportInterceptor Interceptor to for transport requests. Used + * to mock transport layer. + * @param nodeSettings node override of setting + * @param setting the supported setting set. + */ + public void setupTestNodes(TransportInterceptor transportInterceptor, final Settings nodeSettings, Setting... setting) { nodesCount = randomIntBetween(2, 10); testNodes = new FakeNode[nodesCount]; + Set> settingSet = new HashSet<>(Arrays.asList(setting)); for (int i = 0; i < testNodes.length; i++) { - testNodes[i] = new FakeNode("node" + i, threadPool, settings, transportInterceptor); + testNodes[i] = new FakeNode("node" + i, threadPool, nodeSettings, settingSet, transportInterceptor); } FakeNode.connectNodes(testNodes); } - public void setupTestNodes(Settings settings) { - setupTestNodes(settings, TransportService.NOOP_TRANSPORT_INTERCEPTOR); + public void setupTestNodes(Setting... setting) { + setupTestNodes(TransportService.NOOP_TRANSPORT_INTERCEPTOR, Settings.EMPTY, setting); + } + + public void setupTestNodes(Settings nodeSettings) { + setupTestNodes(TransportService.NOOP_TRANSPORT_INTERCEPTOR, nodeSettings); + } + + public void setupTestNodes(TransportInterceptor transportInterceptor) { + setupTestNodes(transportInterceptor, Settings.EMPTY); } public void tearDownTestNodes() { @@ -340,7 +360,7 @@ public HttpRequest releaseAndCopy() { }, null); } - protected boolean areEqualWithArrayValue(Map first, Map second) { + protected boolean areEqualWithArrayValue(Map first, Map second) { if (first.size() != second.size()) { return false; } diff --git a/src/test/java/org/opensearch/ad/transport/DeleteTests.java b/src/test/java/org/opensearch/ad/transport/DeleteTests.java index c35033e70..f4f8f7cb9 100644 --- a/src/test/java/org/opensearch/ad/transport/DeleteTests.java +++ b/src/test/java/org/opensearch/ad/transport/DeleteTests.java @@ -58,7 +58,7 @@ import org.opensearch.ad.AbstractADTest; import org.opensearch.ad.common.exception.JsonPathNotFoundException; import org.opensearch.ad.constant.CommonErrorMessages; -import org.opensearch.ad.constant.CommonMessageAttributes; +import org.opensearch.ad.constant.CommonName; import org.opensearch.ad.util.DiscoveryNodeFilterer; import org.opensearch.client.Client; import org.opensearch.cluster.ClusterName; @@ -141,7 +141,7 @@ public void setUp() throws Exception { transportService = mock(TransportService.class); threadPool = mock(ThreadPool.class); actionFilters = mock(ActionFilters.class); - Settings settings = Settings.builder().put("opendistro.anomaly_detection.request_timeout", TimeValue.timeValueSeconds(10)).build(); + Settings settings = Settings.builder().put("plugins.anomaly_detection.request_timeout", TimeValue.timeValueSeconds(10)).build(); task = mock(Task.class); when(task.getId()).thenReturn(1000L); client = mock(Client.class); @@ -201,7 +201,7 @@ public void testJsonRequestTemplate(R request, Supplier> settingsSet, + TransportInterceptor transportInterceptor + ) { final Function boundTransportAddressDiscoveryNodeFunction = address -> { discoveryNode.set(new DiscoveryNode(name, address.publishAddress(), emptyMap(), emptySet(), Version.CURRENT)); return discoveryNode.get(); }; transportService = new TransportService( - settings, + Settings.EMPTY, new MockNioTransport( - settings, + Settings.EMPTY, Version.CURRENT, threadPool, new NetworkService(Collections.emptyList()), @@ -103,7 +112,10 @@ protected TaskManager createTaskManager(Settings settings, ThreadPool threadPool }; transportService.start(); - clusterService = createClusterService(threadPool, discoveryNode.get()); + Set> internalSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + internalSettings.addAll(settingsSet); + ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, internalSettings); + clusterService = createClusterService(threadPool, discoveryNode.get(), clusterSettings); clusterService.addStateApplier(transportService.getTaskManager()); ActionFilters actionFilters = new ActionFilters(emptySet()); transportListTasksAction = new TransportListTasksAction(clusterService, transportService, actionFilters); @@ -111,8 +123,8 @@ protected TaskManager createTaskManager(Settings settings, ThreadPool threadPool transportService.acceptIncomingRequests(); } - public FakeNode(String name, ThreadPool threadPool, Settings settings) { - this(name, threadPool, settings, TransportService.NOOP_TRANSPORT_INTERCEPTOR); + public FakeNode(String name, ThreadPool threadPool, Set> settings) { + this(name, threadPool, Settings.EMPTY, settings, TransportService.NOOP_TRANSPORT_INTERCEPTOR); } public final ClusterService clusterService;