Fixup ITs and binding issues

.github/workflows/gradle.yml

@@ -45,9 +45,9 @@ jobs:
       # Set the vm.max_map_count system property to the minimum required to run Elasticsearch
     - name: Set vm.max_map_count
       run: sudo sysctl -w vm.max_map_count=262144
-    - name: Start Build
+    - name: Build PA and run Unit Tests
       working-directory: ./tmp/pa
       run: ./gradlew build
     - name: Run Integration Tests
       working-directory: ./tmp/pa
-      run: ./gradlew integTest
+      run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster

build.gradle

@@ -178,8 +178,16 @@ import org.ajoberstar.gradle.git.tasks.GitClone
 
 String rcaDir
 
+static def propEnabled(property) {
+    return System.getProperty(property) != null
+}
+
 // The following Gradle tasks are used to create a PA/RCA enabled Elasticsearch cluster
+// Pass the -Dtests.enableIT property to Gradle to run ITs
 task cloneGitRepo(type: GitClone) {
+    onlyIf = {
+        propEnabled("tests.enableIT")
+    }
     rcaDir = Paths.get(getProject().getBuildDir().toString(), "performance-analyzer-rca").toString()
     def destination = file(rcaDir)
     uri = "https://github.com/opendistro-for-elasticsearch/performance-analyzer-rca.git"
@@ -188,19 +196,31 @@ task cloneGitRepo(type: GitClone) {
     enabled = !destination.exists() // to clone only once
 }
 
-task setupEsCluster(type: Exec) {
+task setupEsCluster() {
     dependsOn(cloneGitRepo)
-    workingDir(rcaDir)
-    commandLine './gradlew', 'enableRca'
+    onlyIf = {
+        propEnabled("tests.enableIT")
+    }
     doLast {
+        exec {
+            workingDir(rcaDir)
+            commandLine 'sed', '-i', '""', 's|#webservice-bind-host =|webservice-bind-host = 0.0.0.0|g', 'pa_config/performance-analyzer.properties'
+        }
+        exec {
+            workingDir(rcaDir)
+            commandLine './gradlew', 'enableRca'
+        }
         sleep(5000)
     }
 }
 
 integTestRunner {
+    onlyIf = {
+        propEnabled("tests.enableIT")
+    }
     // add "-Dtests.security.manager=false" to VM options if you want to run integ tests in IntelliJ
     systemProperty 'tests.security.manager', 'false'
-    if (System.getProperty("tests.useDockerCluster").equals("true")) {
+    if (propEnabled("tests.useDockerCluster")) {
         dependsOn(setupEsCluster)
     }
 }

gradle.properties

@@ -8,4 +8,5 @@ systemProp.tests.rest.cluster=localhost:9200
 # The Elasticsearch cluster node communication endpoint
 systemProp.tests.cluster=localhost:9300
 # Whether or not to spin up a new Elasticsearch cluster for integration testing
-systemProp.tests.useDockerCluster=true
+# Comment this out if you don't want a cluster spun up
+systemProp.tests.useDockerCluster=

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerIT.java

@@ -1,5 +1,6 @@
 package com.amazon.opendistro.elasticsearch.performanceanalyzer;
 
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.WaitFor;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -11,23 +12,51 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.*;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 public class PerformanceAnalyzerIT extends ESRestTestCase {
     private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerIT.class);
     private static final int PORT = 9600;
     private static final ObjectMapper mapper = new ObjectMapper();
     private static RestClient paClient;
 
+    // Don't wipe the cluster after test completion
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    // This method is the same as ESRestTestCase#buildClient, but it attempts to connect
+    // to the provided cluster for 1 minute before giving up. This is useful when we spin up
+    // our own Docker cluster on the local node for Integration Testing
+    @Override
+    protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
+        final RestClient[] restClientArr = new RestClient[1];
+        try {
+            WaitFor.waitFor(() -> {
+                try {
+                    restClientArr[0] = super.buildClient(settings, hosts);
+                } catch (Exception e) {
+                    logger.debug("Error building RestClient against hosts {}", hosts, e);
+                    return false;
+                }
+                return true;
+            }, 1, TimeUnit.MINUTES);
+        } catch (Exception e) {
+            throw new IOException(e);
+        }
+        return restClientArr[0];
+    }
+
     @Before
     public void initPaClient() throws IOException {
         String cluster = System.getProperty("tests.rest.cluster");
@@ -65,12 +94,25 @@ public static void ensurePaAndRcaEnabled() throws Exception {
     @Test
     public void checkMetrics() throws Exception {
         ensurePaAndRcaEnabled();
-        Request request = new Request("GET",
-                "/_opendistro/_performanceanalyzer/metrics/?metrics=Disk_Utilization&agg=max&dim=&nodes=all");
-        Response resp = paClient.performRequest(request);
-        Assert.assertEquals(HttpStatus.SC_OK, resp.getStatusLine().getStatusCode());
-        String jsonString = EntityUtils.toString(resp.getEntity());
-        JsonNode root = mapper.readTree(jsonString);
+        final String[] jsonString = new String[1];
+        WaitFor.waitFor(() -> {
+            Request request = new Request("GET",
+                    "/_opendistro/_performanceanalyzer/metrics/?metrics=Disk_Utilization&agg=max&dim=&nodes=all");
+            Response resp = paClient.performRequest(request);
+            Assert.assertEquals(HttpStatus.SC_OK, resp.getStatusLine().getStatusCode());
+            jsonString[0] = EntityUtils.toString(resp.getEntity());
+            JsonNode root = mapper.readTree(jsonString[0]);
+            for (Iterator<JsonNode> it = root.elements(); it.hasNext(); ) {
+                JsonNode entry = it.next();
+                JsonNode data = entry.get(TestUtils.DATA);
+                if (data.get(TestUtils.FIELDS) == null) {
+                    return false;
+                }
+            }
+            return jsonString[0] != null && !jsonString[0].isEmpty();
+        }, 1, TimeUnit.MINUTES);
+        logger.info("jsonString is {}", jsonString[0]);
+        JsonNode root = mapper.readTree(jsonString[0]);
         root.forEach( entry -> {
             JsonNode data = entry.get(TestUtils.DATA);
             Assert.assertEquals(1, data.get(TestUtils.FIELDS).size());

src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/WaitFor.java

@@ -0,0 +1,32 @@
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.util;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * This class allows you to wait at most a specified duration until a condition evaluates to true
+ */
+public class WaitFor {
+    /**
+     * Waits at most the specified time for the given task to evaluate to true
+     * @param task The task which we hope evaluates to true before the time limit
+     * @param maxWait The max amount of time to wait for the task to evaluate for true
+     * @param unit The time unit of the maxWait parameter
+     * @throws Exception If the time limit expires before the task evaluates to true
+     */
+    public static void waitFor(Callable<Boolean> task, long maxWait, TimeUnit unit) throws Exception {
+        long maxWaitMillis = TimeUnit.MILLISECONDS.convert(maxWait, unit);
+        long pollTime = System.currentTimeMillis();
+        long curTime;
+        while (!task.call() && maxWaitMillis >= 0) {
+            curTime = System.currentTimeMillis();
+            maxWaitMillis -= (curTime - pollTime);
+            pollTime = curTime;
+        }
+        if (maxWait < 0) {
+            throw new TimeoutException("WaitFor timed out before task evaluated to true");
+        }
+    }
+}
+