Merge pull request #87 from vamshin/dev-vam

AD opendistro 1.6 support
opendistro-for-elasticsearch · Apr 23, 2020 · 4285d51 · 4285d51
2 parents e5b6ce5 + 92094ea
commit 4285d51
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 174 deletions.
diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml
@@ -8,7 +8,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [12]
+        java: [13]
     # Job name
     name: Build Anomaly detection with JDK ${{ matrix.java }}
     # This job runs on Linux

diff --git a/README.md b/README.md
@@ -76,39 +76,12 @@ The **test runner JVM** will start suspended and wait for a debugger to attach t
 
 ### Advanced: Launching multi node clusters locally
 
-Sometimes you need to launch a cluster with more than one ES server process. The `startMultiNode` tasks help with this.
+Sometimes you need to launch a cluster with more than one Elasticsearch server process.
 
-#### All nodes are started and stopped together
+You can do this by running `./gradlew run -PnumNodes=<numberOfNodesYouWant>`
 
-If you need a multi node cluster where all nodes are started together use: 
-
-```
-./gradlew -PnumNodes=2 startMultiNode # to launch 2 nodes
-
-```
-
-If you need a single node cluster use:
-
-```
-./gradlew startMultiNode 
-
-```
-
-#### Nodes join and leave the cluster independently
-
-If you need a multi node cluster (up to 3 nodes) where you'd like to be able to add and kill each node independently use:
-
-```
-./gradlew startSingleNode0
-./gradlew startSingleNode1
-./gradlew startSingleNode2
-```
-
-#### Kill the nodes when you're done!
-
-```
-./gradlew stopMultiNode
-```
+You can also debug a multi-node cluster, by using a combination of above multi-node and debug steps.
+But, you must set up debugger configurations to listen on each port starting from `5005` and increasing by 1 for each node.  
 
 ## Known Issues
 * We have a cold start period whenever we initialize a model, which could happen when we create a new detector or when the cluster restarts and models get restored from snapshots. Currently the detector always return errors during the cold start period. Please ignore these initial errors for now. We are actively working on the fix and will push in the next release.

diff --git a/build.gradle b/build.gradle
@@ -16,9 +16,9 @@
 buildscript {
     ext {
         es_group = "org.elasticsearch"
-        es_version = '7.4.2'
-        es_distribution = 'oss-zip'
+        es_version = '7.6.1'
     }
+
     repositories {
         mavenCentral()
         maven { url "https://plugins.gradle.org/m2/" }
@@ -31,8 +31,9 @@ buildscript {
 }
 
 plugins {
-    id 'nebula.ospackage' version "5.3.0"
+    id 'nebula.ospackage' version "8.2.0" apply false
     id "com.diffplug.gradle.spotless" version "3.26.1"
+    id 'java-library'
 }
 
 repositories {
@@ -42,13 +43,8 @@ repositories {
 }
 
 ext {
-    opendistroVersion = '1.4.0'
+    opendistroVersion = '1.6.0'
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-    if (System.properties['os.name'].toLowerCase().contains('windows')) {
-        job_scheduler_plugin_zip = "file:///${fileTree("src/test/resources/job-scheduler").getSingleFile().absolutePath}"
-    } else {
-        job_scheduler_plugin_zip = "file://${fileTree("src/test/resources/job-scheduler").getSingleFile().absolutePath}"
-    }
 }
 
 version = "${opendistroVersion}.0"
@@ -120,80 +116,6 @@ thirdPartyAudit.enabled = false
 // See package README.md for details on using these tasks.
 def _numNodes = findProperty('numNodes') as Integer ?: 1
 
-def getSeedHosts = { int num ->
-    def _localhost = "127.0.0.1:"
-    def _startPort = 9300
-
-    def _seed_hosts = new StringBuilder()
-    _seed_hosts.append("[")
-
-    (0..< num).each { i ->
-        if (i>0) {
-            _seed_hosts.append(", ")
-        }
-        def _host = "\"" +  _localhost + (_startPort + i) + "\""
-        _seed_hosts.append(_host)
-    }
-
-    _seed_hosts.append("]")
-    _seed_hosts
-}
-
-tasks.create(name : "runMultiNode", type: org.elasticsearch.gradle.test.RunTask) {
-    daemonize = true
-    numNodes = _numNodes
-    // this has to be false otherwise ClusterFormationTasks.groovy will set discovery.seed_providers to file
-    autoSetHostsProvider = false
-    setting 'http.port', '9200-9300'
-    setting 'transport.port', '9300-9400'
-    setting 'discovery.seed_hosts', getSeedHosts(numNodes)
-    clusterName = 'multi-node-run'
-    plugin project.path
-    distribution = es_distribution
-    // Temporary until job-scheduler is published to Maven
-    setupCommand('installPlugin', 'bin/elasticsearch-plugin', 'install', job_scheduler_plugin_zip)
-}
-
-tasks.create(name: "startMultiNode") {
-    if (_numNodes == 1) {
-        dependsOn "runMultiNode#start"
-    } else {
-        (0..<_numNodes).each { n -> dependsOn "runMultiNode#node${n}.start" }
-    }
-}
-
-tasks.create(name : "runSingleNode", type: org.elasticsearch.gradle.test.RunTask) {
-    daemonize = true
-    numNodes = 3
-    // this has to be false otherwise ClusterFormationTasks.groovy will set discovery.seed_providers to file
-    autoSetHostsProvider = false
-    // this has to be false otherwise ClusterFormationTasks.groovy will set cluster.initial_master_nodes to all 3 nodes
-    autoSetInitialMasterNodes = false
-    setting 'http.port', '9200-9300'
-    setting 'transport.port', '9300-9400'
-    setting 'discovery.seed_hosts', getSeedHosts(numNodes)
-    setting 'node.master', true
-    setting 'node.data', true
-    setting 'node.ingest', true
-    // since we want to start one node at a time, we have to provide the node we are going to start first
-    setting 'cluster.initial_master_nodes', "[\"node-0\"]" 
-    clusterName = 'multi-node-run'
-    plugin project.path
-    distribution = es_distribution
-}
-
-(0..2).each { i ->
-    tasks.create(name: "startSingleNode$i") {
-        dependsOn "runSingleNode#node${i}.start"
-    }
-}
-
-task stopMultiNode(type: Exec) {
-    commandLine "bash", "-c", "kill -9 \$(ps aux | grep lastic | grep -v grep | grep -v stopMultiNode | awk '{print \$2}')"
-    sleep(1000)
-    dependsOn "clean"
-}
-
 def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
 es_tmp_dir.mkdirs()
 
@@ -202,30 +124,57 @@ test {
     systemProperty 'tests.security.manager', 'false'
 }
 
-integTestRunner {
-    systemProperty 'tests.security.manager', 'false'
-    systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
-    systemProperty 'tests.locale', 'en'
-    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
-    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
-    doFirst { systemProperty 'cluster.debug', integTestCluster.debug }
-
-    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
-    if (System.getProperty("test.debug") != null) {
-        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+integTest {
+    runner {
+        systemProperty 'tests.security.manager', 'false'
+        systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
+        // The 'doFirst' delays till execution time.
+        doFirst {
+            // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
+            // use longer timeouts for requests.
+            def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
+            systemProperty 'cluster.debug', isDebuggingCluster
+            // Set number of nodes system property to be used in tests
+            systemProperty 'cluster.number_of_nodes', "${_numNodes}"
+            // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
+            // not being written, the waitForAllConditions ensures it's written
+            getClusters().forEach { cluster ->
+                cluster.waitForAllConditions()
+            }
+        }
+
+        // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
+        if (System.getProperty("test.debug") != null) {
+            jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+        }
     }
 }
 
-integTestCluster {
-    distribution = es_distribution
-    // Temporary until job-scheduler is published to Maven
-    setupCommand('installPlugin', 'bin/elasticsearch-plugin', 'install', job_scheduler_plugin_zip)
+testClusters.integTest {
+    testDistribution = "OSS"
+    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
+    if (_numNodes > 1) numberOfNodes = _numNodes
+    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
+    // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
+    // since we also support multi node integration tests we increase debugPort per node
+    if (System.getProperty("es.debug") != null) {
+        def debugPort = 5005
+        nodes.forEach { node ->
+            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
+            debugPort += 1
+        }
+    }
+    plugin(fileTree("src/test/resources/job-scheduler").getSingleFile())
 }
 
 run {
-    distribution = es_distribution
-    // Temporary until job-scheduler is published to Maven
-    setupCommand('installPlugin', 'bin/elasticsearch-plugin', 'install', job_scheduler_plugin_zip)
+    doFirst {
+        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
+        // not being written, the waitForAllConditions ensures it's written
+        getClusters().forEach { cluster ->
+            cluster.waitForAllConditions()
+        }
+    }
 }
 
 evaluationDependsOnChildren()
@@ -294,8 +243,9 @@ check.dependsOn jacocoTestCoverageVerification
 jacocoTestCoverageVerification.dependsOn jacocoTestReport
 
 dependencies {
+    compile "org.elasticsearch:elasticsearch:${es_version}"
     compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}"
-    compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:1.4.0.0"
+    compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:1.6.0.0"
     compile group: 'com.google.guava', name: 'guava', version:'15.0'
     compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
     compile group: 'com.google.code.gson', name: 'gson', version: '2.8.5'
@@ -325,29 +275,6 @@ apply plugin: 'nebula.ospackage'
 
 // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
 afterEvaluate {
-    project.tasks.getByName("run#installOpendistroAnomalyDetectorPlugin").dependsOn("run#installPlugin")
-    project.tasks.getByName("run#installPlugin").dependsOn.remove(project.tasks.getByName("run#installOpendistroAnomalyDetectorPlugin"))
-    project.tasks.getByName("run#installPlugin").dependsOn("run#copyPlugins")
-    project.tasks.getByName("run#start").dependsOn.remove(project.tasks.getByName("run#installPlugin"))
-    project.tasks.getByName("run#start").dependsOn("run#installOpendistroAnomalyDetectorPlugin")
-
-    if (_numNodes == 1) {
-        project.tasks.getByName("runMultiNode#installOpendistroAnomalyDetectorPlugin").dependsOn("runMultiNode#installPlugin")
-        project.tasks.getByName("runMultiNode#installPlugin").dependsOn.remove(project.tasks.getByName("runMultiNode#installOpendistroAnomalyDetectorPlugin"))
-        project.tasks.getByName("runMultiNode#installPlugin").dependsOn("runMultiNode#copyPlugins")
-        project.tasks.getByName("runMultiNode#start").dependsOn.remove(project.tasks.getByName("runMultiNode#installPlugin"))
-        project.tasks.getByName("runMultiNode#start").dependsOn("runMultiNode#installOpendistroAnomalyDetectorPlugin")
-    } else {
-        (0..<_numNodes).each {
-            n ->
-                project.tasks.getByName("runMultiNode#node${n}.installOpendistroAnomalyDetectorPlugin").dependsOn("runMultiNode#node${n}.installPlugin")
-                project.tasks.getByName("runMultiNode#node${n}.installPlugin").dependsOn.remove(project.tasks.getByName("runMultiNode#node${n}.installOpendistroAnomalyDetectorPlugin"))
-                project.tasks.getByName("runMultiNode#node${n}.installPlugin").dependsOn("runMultiNode#node${n}.copyPlugins")
-                project.tasks.getByName("runMultiNode#node${n}.start").dependsOn.remove(project.tasks.getByName("runMultiNode#node${n}.installPlugin"))
-                project.tasks.getByName("runMultiNode#node${n}.start").dependsOn("runMultiNode#node${n}.installOpendistroAnomalyDetectorPlugin")
-        }
-    }
-
     ospackage {
         packageName = "${name}"
         release = isSnapshot ? "0.1" : '1'
@@ -395,16 +322,6 @@ afterEvaluate {
     }
 }
 
-tasks.whenTaskAdded { task ->
-    if (task.name == "integTestCluster#wait") {
-        project.tasks.getByName("integTestCluster#installOpendistroAnomalyDetectorPlugin").dependsOn("integTestCluster#installPlugin")
-        project.tasks.getByName("integTestCluster#installPlugin").dependsOn.remove(project.tasks.getByName("integTestCluster#installOpendistroAnomalyDetectorPlugin"))
-        project.tasks.getByName("integTestCluster#installPlugin").dependsOn("integTestCluster#copyPlugins")
-        project.tasks.getByName("integTestCluster#start").dependsOn.remove(project.tasks.getByName("integTestCluster#installPlugin"))
-        project.tasks.getByName("integTestCluster#start").dependsOn("integTestCluster#installOpendistroAnomalyDetectorPlugin")
-    }
-}
-
 spotless {
     java {
         eclipse().configFile rootProject.file('.eclipseformat.xml')

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.2.1-bin.zip
diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/util/ParseUtils.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/util/ParseUtils.java
@@ -219,12 +219,7 @@ public static AggregatorFactories.Builder parseAggregators(XContentParser parser
                                 );
                             }
 
-                            aggBuilder = parser
-                                .namedObject(
-                                    BaseAggregationBuilder.class,
-                                    fieldName,
-                                    new AggregatorFactories.AggParseContext(aggregationName)
-                                );
+                            aggBuilder = parser.namedObject(BaseAggregationBuilder.class, fieldName, aggregationName);
                     }
                 } else {
                     throw new ParsingException(

diff --git a/src/test/resources/job-scheduler/opendistro-job-scheduler-1.4.0.0-SNAPSHOT.zip b/src/test/resources/job-scheduler/opendistro-job-scheduler-1.4.0.0-SNAPSHOT.zip
diff --git a/src/test/resources/job-scheduler/opendistro-job-scheduler-1.6.0.0.zip b/src/test/resources/job-scheduler/opendistro-job-scheduler-1.6.0.0.zip