From 50b25262ce3863a5c448671867854f54f6a11a03 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Wed, 13 May 2020 09:06:48 -0700 Subject: [PATCH 01/14] ODFE IT Framework POC --- build.gradle | 40 ++++++-- gradle.properties | 3 + .../PerformanceAnalyzerIT.java | 91 +++++++++++++++++++ .../RootCauseAnalyzerIT.java | 36 ++++++++ 4 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 gradle.properties create mode 100644 src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java create mode 100644 src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java diff --git a/build.gradle b/build.gradle index 14d8ac33..f7e069bf 100644 --- a/build.gradle +++ b/build.gradle @@ -26,6 +26,7 @@ buildscript { dependencies { classpath "org.elasticsearch.gradle:build-tools:${es_version}" + classpath 'org.ajoberstar:gradle-git:0.2.3' } } @@ -146,11 +147,6 @@ dependencyLicenses.doFirst { updateShas.updateShas() } -integTestRunner { - // add "-Dtests.security.manager=false" to VM options if you want to run integ tests in IntelliJ - systemProperty 'tests.security.manager', 'false' -} - bundlePlugin { from ("pa_config") { into "pa_config" @@ -170,7 +166,6 @@ bundlePlugin { from ("config/opendistro_performance_analyzer") { into "config" } - } gradle.startParameter.excludedTaskNames += [ "forbiddenApisMain", @@ -178,6 +173,37 @@ gradle.startParameter.excludedTaskNames += [ "forbiddenApisMain", "thirdPartyAudit", "testingConventions"] +import java.nio.file.Paths +import org.ajoberstar.gradle.git.tasks.GitClone + +String rcaDir + +// The following Gradle tasks are used to create a PA/RCA enabled Elasticsearch cluster +task cloneGitRepo(type: GitClone) { + rcaDir = Paths.get(getProject().getBuildDir().toString(), "performance-analyzer-rca").toString() + def destination = file(rcaDir) + uri = "https://github.com/sidheart/performance-analyzer-rca.git" + destinationPath = destination + bare = false + enabled = !destination.exists() // to clone only once +} + +task setupEsCluster(type: Exec) { + dependsOn(cloneGitRepo) + workingDir(rcaDir) + commandLine './gradlew', 'enablePaAndRca' + doLast { + sleep(5000) + } +} + +integTestRunner { + if (System.getProperty("tests.useDockerCluster") != null) { + dependsOn(setupEsCluster) + } + // add "-Dtests.security.manager=false" to VM options if you want to run integ tests in IntelliJ +} + // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name afterEvaluate { ospackage { @@ -240,3 +266,5 @@ afterEvaluate { tasks = ['build', 'buildRpm', 'buildDeb'] } } + +check.dependsOn integTest diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000..8cdd130b --- /dev/null +++ b/gradle.properties @@ -0,0 +1,3 @@ +systemProp.tests.rest.cluster=localhost:9200 +systemProp.tests.cluster=localhost:9300 +systemProp.tests.useDockerCluster=true \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java new file mode 100644 index 00000000..90627bd6 --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java @@ -0,0 +1,91 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpHost; +import org.apache.http.HttpStatus; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class PerformanceAnalyzerIT extends ESRestTestCase { + private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerIT.class); + private static final int PORT = 9600; + private static final ObjectMapper mapper = new ObjectMapper(); + private static RestClient paClient; + + @Before + public void initPaClient() throws IOException { + String cluster = System.getProperty("tests.rest.cluster"); + logger.info("Cluster is {}", cluster); + if (cluster == null) { + throw new RuntimeException("Must specify [tests.rest.cluster] system property with a comma delimited list of [host:port] " + + "to which to send REST requests"); + } + String[] stringUrls = cluster.split(","); + List hosts = Collections.singletonList(buildHttpHost("localhost", PORT)); + logger.info("initializing PerformanceAnalyzer client against {}", hosts); + paClient = buildClient(restClientSettings(), hosts.toArray(new HttpHost[0])); + } + + + public static void ensurePaAndRcaEnabled() throws Exception { + /* + Request request = new Request("POST", "_opendistro/_performanceanalyzer/cluster/config"); + request.setJsonEntity("{\"enabled\": true}"); + Response resp = client().performRequest(request); + assert resp.getStatusLine().getStatusCode() == 200; + request = new Request("POST", "_opendistro/_performanceanalyzer/rca/cluster/config"); + request.setJsonEntity("{\"enabled\": true}"); + resp = client().performRequest(request); + assert resp.getStatusLine().getStatusCode() == 200; + LOG.info("PA INITIAL STATUS {}", EntityUtils.toString(resp.getEntity(), "UTF-8")); + */ + // TODO replace with waitFor with a 1min timeout + for (int i = 0; i < 60; i++) { + Response resp = client().performRequest(new Request("GET", "_opendistro/_performanceanalyzer/cluster/config")); + Map respMap = mapper.readValue(EntityUtils.toString(resp.getEntity(), "UTF-8"), + new TypeReference>(){}); + if (respMap.get("currentPerformanceAnalyzerClusterState").equals(3)) { + break; + } + Thread.sleep(1000L); + } + Response resp = client().performRequest(new Request("GET", "_opendistro/_performanceanalyzer/cluster/config")); + Map respMap = mapper.readValue(EntityUtils.toString(resp.getEntity(), "UTF-8"), + new TypeReference>(){}); + if (!respMap.get("currentPerformanceAnalyzerClusterState").equals(3)) { + throw new Exception("PA and RCA are not enabled on the target cluster!"); + } + } + + @Test + public void checkMetrics() throws Exception { + ensurePaAndRcaEnabled(); + Request request = new Request("GET", + "/_opendistro/_performanceanalyzer/metrics/?metrics=Disk_Utilization&agg=max&dim=&nodes=all"); + Response resp = paClient.performRequest(request); + assert resp.getStatusLine().getStatusCode() == HttpStatus.SC_OK; + LOG.info("PA is emitting metrics!! {}", EntityUtils.toString(resp.getEntity(), "UTF-8")); + paClient.close(); + } + + @AfterClass + public static void closePaClient() throws Exception { + ESRestTestCase.closeClients(); + paClient.close(); + LOG.info("AfterClass has run"); + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java new file mode 100644 index 00000000..69b3de25 --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java @@ -0,0 +1,36 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Map; + +@Ignore +public class RootCauseAnalyzerIT extends ESRestTestCase { + private static final Logger LOG = LogManager.getLogger(RootCauseAnalyzerIT.class); + + @Test + public void ensureRcaEnabled() throws Exception { + Request request = new Request("POST", "_opendistro/_performanceanalyzer/rca/cluster/config"); + request.setJsonEntity("{\"enabled\": true}"); + Response resp = client().performRequest(request); + assert resp.getStatusLine().getStatusCode() == 200; + LOG.info("RCA Initial state {}", EntityUtils.toString(resp.getEntity(), "UTF-8")); + // TODO replace with waitFor + Thread.sleep(10000L); + resp = client().performRequest(new Request("GET", "_opendistro/_performanceanalyzer/rca/cluster/config")); + ObjectMapper mapper = new ObjectMapper(); + Map respMap = mapper.readValue(EntityUtils.toString(resp.getEntity(), "UTF-8"), + new TypeReference>(){}); + assert respMap.get("currentPerformanceAnalyzerClusterState").equals(1); + LOG.info("RCA successfully enabled!!: {}", respMap); + } +} From a0735e945d492f36d9100826371de06e41c14c75 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Thu, 14 May 2020 20:58:26 -0700 Subject: [PATCH 02/14] Testing to see if Dockerstuff is set up --- .github/workflows/gradle.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 47db0e17..ad31c936 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -40,6 +40,11 @@ jobs: - name: Update SHA working-directory: ./tmp/pa run: ./gradlew updateShas + - name: Link docker-compose + run: | + echo DOCKER COMPOSE IS HERE: $(which docker-compose) + ln -s $(which docker-compose) /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/docker-compose + echo IS DOCKER COMPOSE ACTUALLY HERE? $(ls -l /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/) - name: Start Build working-directory: ./tmp/pa - run: ./gradlew build + run: ./gradlew build --info From ba6a61fb0a1aff5c1e0c2b64d3fa6fcdf99d724e Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Fri, 15 May 2020 10:48:20 -0700 Subject: [PATCH 03/14] Modified workflow to set DOCKER_COMPOSE_LOCATION --- .github/workflows/gradle.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index ad31c936..af8b4627 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -45,6 +45,9 @@ jobs: echo DOCKER COMPOSE IS HERE: $(which docker-compose) ln -s $(which docker-compose) /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/docker-compose echo IS DOCKER COMPOSE ACTUALLY HERE? $(ls -l /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/) + export DOCKER_COMPOSE_LOCATION=$(which docker-compose) + echo DOCKER_COMPOSE_LOCATION variable is set to ${DOCKER_COMPOSE_LOCATION} + echo docker-compose version is $(docker-compose -v) - name: Start Build working-directory: ./tmp/pa run: ./gradlew build --info From c0c8170c350c550de0c9275830ae14c60f0b51a5 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Fri, 15 May 2020 13:22:33 -0700 Subject: [PATCH 04/14] Modify workflow to include stacktrace and no symbolic linkage --- .github/workflows/gradle.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index af8b4627..fcd39aeb 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -43,11 +43,10 @@ jobs: - name: Link docker-compose run: | echo DOCKER COMPOSE IS HERE: $(which docker-compose) - ln -s $(which docker-compose) /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/docker-compose echo IS DOCKER COMPOSE ACTUALLY HERE? $(ls -l /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/) export DOCKER_COMPOSE_LOCATION=$(which docker-compose) echo DOCKER_COMPOSE_LOCATION variable is set to ${DOCKER_COMPOSE_LOCATION} echo docker-compose version is $(docker-compose -v) - name: Start Build working-directory: ./tmp/pa - run: ./gradlew build --info + run: ./gradlew build --info --stacktrace From 93f1dafde5749e6d48167f0ea1939790957d2a64 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Fri, 15 May 2020 13:34:49 -0700 Subject: [PATCH 05/14] Set DOCKER_COMPOSE_LOCATION using set-env --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index fcd39aeb..aca582ff 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -42,9 +42,9 @@ jobs: run: ./gradlew updateShas - name: Link docker-compose run: | + echo "::set-env name=DOCKER_COMPOSE_LOCATION::$(which docker-compose)" echo DOCKER COMPOSE IS HERE: $(which docker-compose) echo IS DOCKER COMPOSE ACTUALLY HERE? $(ls -l /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/) - export DOCKER_COMPOSE_LOCATION=$(which docker-compose) echo DOCKER_COMPOSE_LOCATION variable is set to ${DOCKER_COMPOSE_LOCATION} echo docker-compose version is $(docker-compose -v) - name: Start Build From e0f101e5fd4cfba3b02305566021490bcc60902c Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Fri, 15 May 2020 13:48:22 -0700 Subject: [PATCH 06/14] Try set-env in a different location --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index aca582ff..259b0afd 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -20,6 +20,7 @@ jobs: with: repository: opendistro-for-elasticsearch/performance-analyzer-rca path: ./tmp/rca + run: echo ::set-env name=DOCKER_COMPOSE_LOCATION::$(which docker-compose) - name: Checkout Performance Analyzer package uses: actions/checkout@v2 with: @@ -42,7 +43,6 @@ jobs: run: ./gradlew updateShas - name: Link docker-compose run: | - echo "::set-env name=DOCKER_COMPOSE_LOCATION::$(which docker-compose)" echo DOCKER COMPOSE IS HERE: $(which docker-compose) echo IS DOCKER COMPOSE ACTUALLY HERE? $(ls -l /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/) echo DOCKER_COMPOSE_LOCATION variable is set to ${DOCKER_COMPOSE_LOCATION} From 04d1703760f9ffc8a18c41e99366de395d613d0a Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Fri, 15 May 2020 14:00:58 -0700 Subject: [PATCH 07/14] Attempt to fix docker-compose set-env --- .github/workflows/gradle.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 259b0afd..451e5495 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -20,7 +20,6 @@ jobs: with: repository: opendistro-for-elasticsearch/performance-analyzer-rca path: ./tmp/rca - run: echo ::set-env name=DOCKER_COMPOSE_LOCATION::$(which docker-compose) - name: Checkout Performance Analyzer package uses: actions/checkout@v2 with: @@ -41,6 +40,8 @@ jobs: - name: Update SHA working-directory: ./tmp/pa run: ./gradlew updateShas + - name: Set DOCKER_COMPOSE_LOCATION + run: echo ::set-env name=DOCKER_COMPOSE_LOCATION::$(which docker-compose) - name: Link docker-compose run: | echo DOCKER COMPOSE IS HERE: $(which docker-compose) From 4d811f4b2118180ce47578d178a6c7a039933a9a Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Mon, 18 May 2020 09:07:10 -0700 Subject: [PATCH 08/14] Make workflow set vm.max_map_count --- .github/workflows/gradle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 451e5495..79c000b2 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -48,6 +48,8 @@ jobs: echo IS DOCKER COMPOSE ACTUALLY HERE? $(ls -l /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/) echo DOCKER_COMPOSE_LOCATION variable is set to ${DOCKER_COMPOSE_LOCATION} echo docker-compose version is $(docker-compose -v) + - name: Set vm.max_map_count + run: sysctl -w vm.max_map_count=262144 - name: Start Build working-directory: ./tmp/pa run: ./gradlew build --info --stacktrace From 7d37c72b5557f4f59b03d93ce57633944b0c4a27 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Mon, 18 May 2020 10:33:30 -0700 Subject: [PATCH 09/14] Use sudo when setting vm.max_map_count --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 79c000b2..cd628551 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -49,7 +49,7 @@ jobs: echo DOCKER_COMPOSE_LOCATION variable is set to ${DOCKER_COMPOSE_LOCATION} echo docker-compose version is $(docker-compose -v) - name: Set vm.max_map_count - run: sysctl -w vm.max_map_count=262144 + run: sudo sysctl -w vm.max_map_count=262144 - name: Start Build working-directory: ./tmp/pa run: ./gradlew build --info --stacktrace From 321c4ad85d41f22436b6a252109ae9d3551af035 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Tue, 19 May 2020 10:16:53 -0700 Subject: [PATCH 10/14] Make performance-analyzer execute integration tests on checkin --- .github/workflows/gradle.yml | 9 +-------- build.gradle | 5 +++-- gradle.properties | 8 ++++++++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index cd628551..58d79897 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -8,7 +8,6 @@ on: pull_request: branches: - master - jobs: build_rca_pkg: @@ -40,14 +39,8 @@ jobs: - name: Update SHA working-directory: ./tmp/pa run: ./gradlew updateShas - - name: Set DOCKER_COMPOSE_LOCATION + - name: Set docker-compose path run: echo ::set-env name=DOCKER_COMPOSE_LOCATION::$(which docker-compose) - - name: Link docker-compose - run: | - echo DOCKER COMPOSE IS HERE: $(which docker-compose) - echo IS DOCKER COMPOSE ACTUALLY HERE? $(ls -l /home/runner/work/performance-analyzer/performance-analyzer/tmp/pa/build/performance-analyzer-rca/build/docker-build/rca-docker/) - echo DOCKER_COMPOSE_LOCATION variable is set to ${DOCKER_COMPOSE_LOCATION} - echo docker-compose version is $(docker-compose -v) - name: Set vm.max_map_count run: sudo sysctl -w vm.max_map_count=262144 - name: Start Build diff --git a/build.gradle b/build.gradle index f7e069bf..03d14d60 100644 --- a/build.gradle +++ b/build.gradle @@ -198,10 +198,11 @@ task setupEsCluster(type: Exec) { } integTestRunner { - if (System.getProperty("tests.useDockerCluster") != null) { + // add "-Dtests.security.manager=false" to VM options if you want to run integ tests in IntelliJ + systemProperty 'tests.security.manager', 'false' + if (System.getProperty("tests.useDockerCluster").equals("true")) { dependsOn(setupEsCluster) } - // add "-Dtests.security.manager=false" to VM options if you want to run integ tests in IntelliJ } // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name diff --git a/gradle.properties b/gradle.properties index 8cdd130b..234dc8ae 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,11 @@ +# The following 3 properties are used for integration testing with Elasticsearch +# The "enablePaAndRca" gradle task in the performance-analyzer-rca repository contains logic to spin up a 2-node +# Elasticsearch cluster with the PA and RCA components enabled. The cluster endpoint for this cluster is localhost:9300 +# and the REST endpoint is localhost:9200. + +# The Elasticsearch cluster endpoint to use for test REST requests systemProp.tests.rest.cluster=localhost:9200 +# The Elasticsearch cluster node communication endpoint systemProp.tests.cluster=localhost:9300 +# Whether or not to spin up a new Elasticsearch cluster for integration testing systemProp.tests.useDockerCluster=true \ No newline at end of file From 9f3938ef9602cc239ebf872f8537c59324444fef Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Wed, 20 May 2020 16:54:51 -0700 Subject: [PATCH 11/14] Clean up PerformanceAnalyzerIT and build.gradle script --- build.gradle | 4 +-- .../PerformanceAnalyzerIT.java | 19 +++------- .../RootCauseAnalyzerIT.java | 36 ------------------- 3 files changed, 6 insertions(+), 53 deletions(-) delete mode 100644 src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java diff --git a/build.gradle b/build.gradle index 03d14d60..f00f4e48 100644 --- a/build.gradle +++ b/build.gradle @@ -182,7 +182,7 @@ String rcaDir task cloneGitRepo(type: GitClone) { rcaDir = Paths.get(getProject().getBuildDir().toString(), "performance-analyzer-rca").toString() def destination = file(rcaDir) - uri = "https://github.com/sidheart/performance-analyzer-rca.git" + uri = "https://github.com/opendistro-for-elasticsearch/performance-analyzer-rca.git" destinationPath = destination bare = false enabled = !destination.exists() // to clone only once @@ -191,7 +191,7 @@ task cloneGitRepo(type: GitClone) { task setupEsCluster(type: Exec) { dependsOn(cloneGitRepo) workingDir(rcaDir) - commandLine './gradlew', 'enablePaAndRca' + commandLine './gradlew', 'enableRca' doLast { sleep(5000) } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java index 90627bd6..43d7b332 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java @@ -34,25 +34,14 @@ public void initPaClient() throws IOException { throw new RuntimeException("Must specify [tests.rest.cluster] system property with a comma delimited list of [host:port] " + "to which to send REST requests"); } - String[] stringUrls = cluster.split(","); - List hosts = Collections.singletonList(buildHttpHost("localhost", PORT)); + List hosts = Collections.singletonList( + buildHttpHost(cluster.substring(0, cluster.lastIndexOf(":")), PORT)); logger.info("initializing PerformanceAnalyzer client against {}", hosts); paClient = buildClient(restClientSettings(), hosts.toArray(new HttpHost[0])); } public static void ensurePaAndRcaEnabled() throws Exception { - /* - Request request = new Request("POST", "_opendistro/_performanceanalyzer/cluster/config"); - request.setJsonEntity("{\"enabled\": true}"); - Response resp = client().performRequest(request); - assert resp.getStatusLine().getStatusCode() == 200; - request = new Request("POST", "_opendistro/_performanceanalyzer/rca/cluster/config"); - request.setJsonEntity("{\"enabled\": true}"); - resp = client().performRequest(request); - assert resp.getStatusLine().getStatusCode() == 200; - LOG.info("PA INITIAL STATUS {}", EntityUtils.toString(resp.getEntity(), "UTF-8")); - */ // TODO replace with waitFor with a 1min timeout for (int i = 0; i < 60; i++) { Response resp = client().performRequest(new Request("GET", "_opendistro/_performanceanalyzer/cluster/config")); @@ -79,13 +68,13 @@ public void checkMetrics() throws Exception { Response resp = paClient.performRequest(request); assert resp.getStatusLine().getStatusCode() == HttpStatus.SC_OK; LOG.info("PA is emitting metrics!! {}", EntityUtils.toString(resp.getEntity(), "UTF-8")); - paClient.close(); + System.out.println("WASSUP"); } @AfterClass public static void closePaClient() throws Exception { ESRestTestCase.closeClients(); paClient.close(); - LOG.info("AfterClass has run"); + LOG.debug("AfterClass has run"); } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java deleted file mode 100644 index 69b3de25..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/RootCauseAnalyzerIT.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.amazon.opendistro.elasticsearch.performanceanalyzer; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.util.EntityUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.junit.Ignore; -import org.junit.Test; - -import java.util.Map; - -@Ignore -public class RootCauseAnalyzerIT extends ESRestTestCase { - private static final Logger LOG = LogManager.getLogger(RootCauseAnalyzerIT.class); - - @Test - public void ensureRcaEnabled() throws Exception { - Request request = new Request("POST", "_opendistro/_performanceanalyzer/rca/cluster/config"); - request.setJsonEntity("{\"enabled\": true}"); - Response resp = client().performRequest(request); - assert resp.getStatusLine().getStatusCode() == 200; - LOG.info("RCA Initial state {}", EntityUtils.toString(resp.getEntity(), "UTF-8")); - // TODO replace with waitFor - Thread.sleep(10000L); - resp = client().performRequest(new Request("GET", "_opendistro/_performanceanalyzer/rca/cluster/config")); - ObjectMapper mapper = new ObjectMapper(); - Map respMap = mapper.readValue(EntityUtils.toString(resp.getEntity(), "UTF-8"), - new TypeReference>(){}); - assert respMap.get("currentPerformanceAnalyzerClusterState").equals(1); - LOG.info("RCA successfully enabled!!: {}", respMap); - } -} From 68da6150f877559582a8683a514770a4dfe30d13 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Wed, 20 May 2020 16:59:19 -0700 Subject: [PATCH 12/14] Add newline to end of gradle.properties file --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 234dc8ae..c7283048 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,4 +8,4 @@ systemProp.tests.rest.cluster=localhost:9200 # The Elasticsearch cluster node communication endpoint systemProp.tests.cluster=localhost:9300 # Whether or not to spin up a new Elasticsearch cluster for integration testing -systemProp.tests.useDockerCluster=true \ No newline at end of file +systemProp.tests.useDockerCluster=true From 998f7bc6f09915bc8e6f394337da9e6a3ae0ccfa Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Tue, 26 May 2020 16:40:08 -0700 Subject: [PATCH 13/14] Modify gradle.yml and checkMetrics --- .github/workflows/gradle.yml | 5 ++- .../PerformanceAnalyzerIT.java | 33 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 58d79897..09d3d630 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -39,10 +39,13 @@ jobs: - name: Update SHA working-directory: ./tmp/pa run: ./gradlew updateShas + # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program + # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment - name: Set docker-compose path run: echo ::set-env name=DOCKER_COMPOSE_LOCATION::$(which docker-compose) + # Set the vm.max_map_count system property to the minimum required to run Elasticsearch - name: Set vm.max_map_count run: sudo sysctl -w vm.max_map_count=262144 - name: Start Build working-directory: ./tmp/pa - run: ./gradlew build --info --stacktrace + run: ./gradlew build diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java index 43d7b332..9753040b 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java @@ -1,6 +1,7 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.http.HttpHost; import org.apache.http.HttpStatus; @@ -12,6 +13,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -60,15 +62,40 @@ public static void ensurePaAndRcaEnabled() throws Exception { } } + private static class TestUtils { + public static final String DATA = "data"; + public static final String RECORDS = "records"; + + // Field related strings + public static final String FIELDS = "fields"; + public static final String FIELD_NAME = "name"; + public static final String FIELD_TYPE = "type"; + public static final String DOUBLE_TYPE = "DOUBLE"; + + // Metrics related strings + public static final String M_DISKUTIL = "Disk_Utilization"; + } + @Test public void checkMetrics() throws Exception { ensurePaAndRcaEnabled(); Request request = new Request("GET", "/_opendistro/_performanceanalyzer/metrics/?metrics=Disk_Utilization&agg=max&dim=&nodes=all"); Response resp = paClient.performRequest(request); - assert resp.getStatusLine().getStatusCode() == HttpStatus.SC_OK; - LOG.info("PA is emitting metrics!! {}", EntityUtils.toString(resp.getEntity(), "UTF-8")); - System.out.println("WASSUP"); + Assert.assertEquals(HttpStatus.SC_OK, resp.getStatusLine().getStatusCode()); + ObjectMapper mapper = new ObjectMapper(); + String jsonString = EntityUtils.toString(resp.getEntity()); + JsonNode root = mapper.readTree(jsonString); + root.forEach( entry -> { + JsonNode data = entry.get(TestUtils.DATA); + Assert.assertEquals(1, data.get(TestUtils.FIELDS).size()); + JsonNode field = data.get(TestUtils.FIELDS).get(0); + Assert.assertEquals(TestUtils.M_DISKUTIL, field.get(TestUtils.FIELD_NAME).asText()); + Assert.assertEquals(TestUtils.DOUBLE_TYPE, field.get(TestUtils.FIELD_TYPE).asText()); + JsonNode records = data.get(TestUtils.RECORDS); + Assert.assertEquals(1, records.size()); + records.get(0).forEach(record -> Assert.assertTrue(record.asDouble() >= 0)); + }); } @AfterClass From 30e106c327c1cb3c203ca9976202fa6861bd6d69 Mon Sep 17 00:00:00 2001 From: Sid Narayan Date: Mon, 15 Jun 2020 11:50:30 -0700 Subject: [PATCH 14/14] Fix ObjectMapper allocation and move TestUtils definition --- .../PerformanceAnalyzerIT.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java index 9753040b..38265250 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java @@ -62,20 +62,6 @@ public static void ensurePaAndRcaEnabled() throws Exception { } } - private static class TestUtils { - public static final String DATA = "data"; - public static final String RECORDS = "records"; - - // Field related strings - public static final String FIELDS = "fields"; - public static final String FIELD_NAME = "name"; - public static final String FIELD_TYPE = "type"; - public static final String DOUBLE_TYPE = "DOUBLE"; - - // Metrics related strings - public static final String M_DISKUTIL = "Disk_Utilization"; - } - @Test public void checkMetrics() throws Exception { ensurePaAndRcaEnabled(); @@ -83,7 +69,6 @@ public void checkMetrics() throws Exception { "/_opendistro/_performanceanalyzer/metrics/?metrics=Disk_Utilization&agg=max&dim=&nodes=all"); Response resp = paClient.performRequest(request); Assert.assertEquals(HttpStatus.SC_OK, resp.getStatusLine().getStatusCode()); - ObjectMapper mapper = new ObjectMapper(); String jsonString = EntityUtils.toString(resp.getEntity()); JsonNode root = mapper.readTree(jsonString); root.forEach( entry -> { @@ -104,4 +89,18 @@ public static void closePaClient() throws Exception { paClient.close(); LOG.debug("AfterClass has run"); } + + private static class TestUtils { + public static final String DATA = "data"; + public static final String RECORDS = "records"; + + // Field related strings + public static final String FIELDS = "fields"; + public static final String FIELD_NAME = "name"; + public static final String FIELD_TYPE = "type"; + public static final String DOUBLE_TYPE = "DOUBLE"; + + // Metrics related strings + public static final String M_DISKUTIL = "Disk_Utilization"; + } }