Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrating plugin to work with OpenSearch #1

Merged
merged 1 commit into from
Apr 8, 2021
Merged

Migrating plugin to work with OpenSearch #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
58 changes: 36 additions & 22 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,29 @@
*/

import java.util.concurrent.Callable
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.opensearch.gradle.test.RestIntegTestTask

buildscript {
ext {
es_group = "org.elasticsearch"
es_version = '7.10.2'
opensearch_group = "org.opensearch"
opensearch_version = '7.10.3'
}

repositories {
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
maven {
url = 's3://search-vemsarat/'
credentials(AwsCredentials) {
accessKey = System.env.AWS_ACCESS_KEY_ID ?: findProperty('aws_access_key_id')
secretKey = System.env.AWS_SECRET_ACCESS_KEY_ID ?: findProperty('aws_secret_access_key')
}
}
jcenter()
}

dependencies {
classpath "${es_group}.gradle:build-tools:${es_version}"
classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}-SNAPSHOT"
}
}

Expand All @@ -43,20 +50,27 @@ plugins {
repositories {
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
maven {
url = 's3://search-vemsarat/'
credentials(AwsCredentials) {
accessKey = System.env.AWS_ACCESS_KEY_ID ?: findProperty('aws_access_key_id')
secretKey = System.env.AWS_SECRET_ACCESS_KEY_ID ?: findProperty('aws_secret_access_key')
}
}
jcenter()
}

ext {
opendistroVersion = '1.13.0'
opendistroVersion = '1.15.0'
weicongs-amazon marked this conversation as resolved.
Show resolved Hide resolved
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

version = "${opendistroVersion}.0"

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'elasticsearch.esplugin'
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.testclusters'
apply plugin: 'base'
apply plugin: 'jacoco'
apply plugin: 'eclipse'
Expand All @@ -75,7 +89,7 @@ ext {
noticeFile = rootProject.file('NOTICE.txt')
}

esplugin {
opensearchplugin {
name 'opendistro-anomaly-detection'
description 'Amazon opendistro elasticsearch anomaly detector plugin'
classname 'com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin'
Expand Down Expand Up @@ -133,8 +147,8 @@ loggerUsageCheck.enabled = false
// See package README.md for details on using these tasks.
def _numNodes = findProperty('numNodes') as Integer ?: 1

def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
es_tmp_dir.mkdirs()
def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

test {
include '**/*Tests.class'
Expand All @@ -151,7 +165,7 @@ tasks.named("check").configure { dependsOn(integTest) }
integTest {
dependsOn "bundlePlugin"
systemProperty 'tests.security.manager', 'false'
systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath

systemProperty "https", System.getProperty("https")
systemProperty "user", System.getProperty("user")
Expand Down Expand Up @@ -200,7 +214,7 @@ testClusters.integTest {
// When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
// i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
// since we also support multi node integration tests we increase debugPort per node
if (System.getProperty("es.debug") != null) {
if (System.getProperty("opensearch.debug") != null) {
def debugPort = 5005
nodes.forEach { node ->
node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
Expand All @@ -225,7 +239,7 @@ testClusters.integTest {
// the opendistro-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug.
//
// Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opendistro-job-scheduler], dependency of [opendistro-anomaly-detection]
// at org.elasticsearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452)
// at org.opensearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452)
//
// One explanation is that ES build script sort plugins according to the natural ordering of their names.
// opendistro-anomaly-detection comes before opendistro-job-scheduler.
Expand Down Expand Up @@ -265,8 +279,8 @@ task release(type: Copy, group: 'build') {
from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile()))
into "build/plugins/opendistro-anomaly-detection"
includeEmptyDirs = false
// ES versions < 6.3 have a top-level elasticsearch directory inside the plugin zip which we need to remove
eachFile { it.path = it.path - "elasticsearch/" }
// ES versions < 6.3 have a top-level opensearch directory inside the plugin zip which we need to remove
eachFile { it.path = it.path - "opensearch/" }
}

List<String> jacocoExclusions = [
Expand Down Expand Up @@ -351,9 +365,9 @@ checkstyle {
}

dependencies {
compile "org.elasticsearch:elasticsearch:${es_version}"
compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}"
compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:${opendistroVersion}.0"
compile "org.opensearch:opensearch:${opensearch_version}-SNAPSHOT"
compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}"
compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:${opendistroVersion}.0-SNAPSHOT"
compile "com.amazon.opendistroforelasticsearch:common-utils:${opendistroVersion}.0"
compile group: 'com.google.guava', name: 'guava', version:'29.0-jre'
compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
Expand All @@ -363,7 +377,7 @@ dependencies {
compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
compile 'software.amazon.randomcutforest:randomcutforest-core:1.0'
compile 'software.amazon.randomcutforest:randomcutforest-serialization-json:1.0'
compile "org.elasticsearch.client:elasticsearch-rest-client:${es_version}"
compile "org.opensearch.client:opensearch-rest-client:${opensearch_version}-SNAPSHOT"

compile "org.jacoco:org.jacoco.agent:0.8.5"
compile ("org.jacoco:org.jacoco.ant:0.8.5") {
Expand Down Expand Up @@ -399,17 +413,17 @@ afterEvaluate {
release = isSnapshot ? "0.1" : '1'
version = "${project.version}" - "-SNAPSHOT"

into '/usr/share/elasticsearch/plugins'
into '/usr/share/opensearch/plugins'
from(zipTree(bundlePlugin.archivePath)) {
into esplugin.name
into opensearchplugin.name
}

user 'root'
permissionGroup 'root'
fileMode 0644
dirMode 0755

requires('elasticsearch-oss', versions.elasticsearch, EQUAL)
requires('opensearch-oss', versions.opensearch, EQUAL)
weicongs-amazon marked this conversation as resolved.
Show resolved Hide resolved
packager = 'Amazon'
vendor = 'Amazon'
os = 'LINUX'
Expand Down
34 changes: 17 additions & 17 deletions src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorJobRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will we change the package name from "opendistroforelasticsearch" to "opensearch" ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Renaming opendistro has lot more changes needed.
Example: 1. APIs, indices, etc.

So we are holding that off, and working on a plan to cleanly migrate opendistro to opensearch.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it. Make sense.

import static com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin.AD_THREAD_POOL_NAME;
import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE;
import static org.elasticsearch.action.DocWriteResponse.Result.CREATED;
import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import static org.opensearch.action.DocWriteResponse.Result.CREATED;
import static org.opensearch.action.DocWriteResponse.Result.UPDATED;
import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;

import java.io.IOException;
import java.time.Instant;
Expand All @@ -30,20 +30,20 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.threadpool.ThreadPool;
import org.opensearch.action.ActionListener;
import org.opensearch.action.get.GetRequest;
import org.opensearch.action.get.GetResponse;
import org.opensearch.action.index.IndexRequest;
import org.opensearch.action.index.IndexResponse;
import org.opensearch.action.support.WriteRequest;
import org.opensearch.client.Client;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.NamedXContentRegistry;
import org.opensearch.common.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.threadpool.ThreadPool;

import com.amazon.opendistroforelasticsearch.ad.common.exception.AnomalyDetectionException;
import com.amazon.opendistroforelasticsearch.ad.common.exception.EndRunException;
Expand Down
68 changes: 34 additions & 34 deletions src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,38 +30,38 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.SpecialPermission;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.monitor.jvm.JvmService;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.ScriptPlugin;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ExecutorBuilder;
import org.elasticsearch.threadpool.ScalingExecutorBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.opensearch.SpecialPermission;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.IndexScopedSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsFilter;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
import org.opensearch.common.xcontent.NamedXContentRegistry;
import org.opensearch.common.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentParserUtils;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.monitor.jvm.JvmService;
import org.opensearch.plugins.ActionPlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.plugins.ScriptPlugin;
import org.opensearch.repositories.RepositoriesService;
import org.opensearch.rest.RestController;
import org.opensearch.rest.RestHandler;
import org.opensearch.script.ScriptService;
import org.opensearch.threadpool.ExecutorBuilder;
import org.opensearch.threadpool.ScalingExecutorBuilder;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

import com.amazon.opendistroforelasticsearch.ad.breaker.ADCircuitBreakerService;
import com.amazon.opendistroforelasticsearch.ad.caching.CacheProvider;
Expand Down Expand Up @@ -604,14 +604,14 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
new ScalingExecutorBuilder(
AD_THREAD_POOL_NAME,
1,
Math.max(1, EsExecutors.allocatedProcessors(settings) / 4),
Math.max(1, OpenSearchExecutors.allocatedProcessors(settings) / 4),
TimeValue.timeValueMinutes(10),
AD_THREAD_POOL_PREFIX + AD_THREAD_POOL_NAME
),
new ScalingExecutorBuilder(
AD_BATCH_TASK_THREAD_POOL_NAME,
1,
Math.max(1, EsExecutors.allocatedProcessors(settings) / 8),
Math.max(1, OpenSearchExecutors.allocatedProcessors(settings) / 8),
TimeValue.timeValueMinutes(10),
AD_THREAD_POOL_PREFIX + AD_BATCH_TASK_THREAD_POOL_NAME
)
Expand Down
44 changes: 22 additions & 22 deletions src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import static com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX;
import static com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;

import java.io.IOException;
import java.security.InvalidParameterException;
Expand All @@ -29,27 +29,27 @@
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.util.Throwables;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParseException;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.InternalCardinality;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.transport.TransportService;
import org.opensearch.action.ActionListener;
import org.opensearch.action.get.GetRequest;
import org.opensearch.action.get.GetResponse;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.NamedXContentRegistry;
import org.opensearch.common.xcontent.XContentParseException;
import org.opensearch.common.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.search.SearchHits;
import org.opensearch.search.aggregations.Aggregation;
import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder;
import org.opensearch.search.aggregations.metrics.InternalCardinality;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.transport.TransportService;

import com.amazon.opendistroforelasticsearch.ad.common.exception.ResourceNotFoundException;
import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages;
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.ActionListener;
import org.opensearch.action.ActionListener;

import com.amazon.opendistroforelasticsearch.ad.constant.CommonValue;
import com.amazon.opendistroforelasticsearch.ad.feature.FeatureManager;
Expand Down
Loading