[CCR] Restructured QA modules (#36404)

Renamed the follow qa modules:
`multi-cluster-downgraded-to-basic-license` to `downgraded-to-basic-license`
`multi-cluster-with-non-compliant-license` to `non-compliant-license`
`multi-cluster-with-security` to `security`

Moved the `chain` module into the `multi-cluster` module and
changed the `multi-cluster` to start 3 clusters.

Followup from #36031

x-pack/plugin/ccr/qa/multi-cluster/build.gradle

@@ -16,27 +16,48 @@ leaderClusterTestCluster {
     numNodes = 1
     clusterName = 'leader-cluster'
     setting 'xpack.license.self_generated.type', 'trial'
+    setting 'node.name', 'leader'
 }
 
 leaderClusterTestRunner {
     systemProperty 'tests.target_cluster', 'leader'
 }
 
+task middleClusterTest(type: RestIntegTestTask) {}
+
+middleClusterTestCluster {
+    dependsOn leaderClusterTestRunner
+    numNodes = 1
+    clusterName = 'middle-cluster'
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
+    setting 'node.name', 'middle'
+}
+
+middleClusterTestRunner {
+    systemProperty 'tests.target_cluster', 'middle'
+    systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
+}
+
 task followClusterTest(type: RestIntegTestTask) {}
 
 followClusterTestCluster {
-    dependsOn leaderClusterTestRunner
+    dependsOn middleClusterTestRunner
     numNodes = 1
     clusterName = 'follow-cluster'
     setting 'xpack.monitoring.collection.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
+    setting 'cluster.remote.middle_cluster.seeds', "\"${-> middleClusterTest.nodes.get(0).transportUri()}\""
+    setting 'node.name', 'follow'
 }
 
 followClusterTestRunner {
     systemProperty 'tests.target_cluster', 'follow'
     systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
+    systemProperty 'tests.middle_host', "${-> middleClusterTest.nodes.get(0).httpUri()}"
     finalizedBy 'leaderClusterTestCluster#stop'
+    finalizedBy 'middleClusterTestCluster#stop'
 }
 
 check.dependsOn followClusterTest

x-pack/plugin/ccr/qa/chain/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java → x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java

@@ -11,16 +11,21 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 
+import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.equalTo;
 
 public class AutoFollowIT extends ESCCRRestTestCase {
 
-    public void testAutoFollowPatterns() throws Exception {
+    public void testMultipleAutoFollowPatternsDifferentClusters() throws Exception {
         if ("follow".equals(targetCluster) == false) {
+            logger.info("skipping test, waiting for target cluster [follow]" );
             return;
         }
+
+        int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices();
         Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/leader_cluster_pattern");
         putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"logs-*\"], \"remote_cluster\": \"leader_cluster\"}");
         assertOK(client().performRequest(putPatternRequest));
@@ -54,10 +59,7 @@ public void testAutoFollowPatterns() throws Exception {
             }
         }
         assertBusy(() -> {
-            Request statsRequest = new Request("GET", "/_ccr/stats");
-            Map<?, ?> response = toMap(client().performRequest(statsRequest));
-            Map<?, ?> autoFollowStats = (Map<?, ?>) response.get("auto_follow_stats");
-            assertThat(autoFollowStats.get("number_of_successful_follow_indices"), equalTo(2));
+            assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 2));
 
             ensureYellow("logs-20190101");
             ensureYellow("logs-20200101");
@@ -66,4 +68,49 @@ public void testAutoFollowPatterns() throws Exception {
         });
     }
 
+    public void testAutoFollowPatterns() throws Exception {
+        if ("follow".equals(targetCluster) == false) {
+            logger.info("skipping test, waiting for target cluster [follow]" );
+            return;
+        }
+
+        int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices();
+        Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern");
+        request.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], \"remote_cluster\": \"leader_cluster\"}");
+        assertOK(client().performRequest(request));
+
+        try (RestClient leaderClient = buildLeaderClient()) {
+            Settings settings = Settings.builder()
+                .put("index.soft_deletes.enabled", true)
+                .build();
+            request = new Request("PUT", "/metrics-20210101");
+            request.setJsonEntity("{\"settings\": " + Strings.toString(settings) +
+                ", \"mappings\": {\"_doc\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}} }");
+            assertOK(leaderClient.performRequest(request));
+
+            for (int i = 0; i < 5; i++) {
+                String id = Integer.toString(i);
+                index(leaderClient, "metrics-20210101", id, "field", i, "filtered_field", "true");
+            }
+        }
+
+        assertBusy(() -> {
+            assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1));
+            ensureYellow("metrics-20210101");
+            verifyDocuments("metrics-20210101", 5, "filtered_field:true");
+        });
+        assertBusy(() -> {
+            verifyCcrMonitoring("metrics-20210101", "metrics-20210101");
+            verifyAutoFollowMonitoring();
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    private int getNumberOfSuccessfulFollowedIndices() throws IOException {
+        Request statsRequest = new Request("GET", "/_ccr/stats");
+        Map<?, ?> response = toMap(client().performRequest(statsRequest));
+        response = (Map<?, ?>) response.get("auto_follow_stats");
+        return (Integer) response.get("number_of_successful_follow_indices");
+    }
+
+
 }

x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java

@@ -8,10 +8,8 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 
-import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.containsString;
@@ -44,7 +42,7 @@ public void testFollowIndex() throws Exception {
             }
             refresh(leaderIndexName);
             verifyDocuments(leaderIndexName, numDocs, "filtered_field:true");
-        } else {
+        } else if ("follow".equals(targetCluster)) {
             logger.info("Running against follow cluster");
             final String followIndexName = "test_index2";
             followIndex(leaderIndexName, followIndexName);
@@ -70,7 +68,10 @@ public void testFollowIndex() throws Exception {
     }
 
     public void testFollowNonExistingLeaderIndex() throws Exception {
-        assumeFalse("Test should only run when both clusters are running", "leader".equals(targetCluster));
+        if ("follow".equals(targetCluster) == false) {
+            logger.info("skipping test, waiting for target cluster [follow]" );
+            return;
+        }
         ResponseException e = expectThrows(ResponseException.class, () -> resumeFollow("non-existing-index"));
         assertThat(e.getMessage(), containsString("no such index"));
         assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
@@ -80,41 +81,4 @@ public void testFollowNonExistingLeaderIndex() throws Exception {
         assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
     }
 
-    public void testAutoFollowPatterns() throws Exception {
-        assumeFalse("Test should only run when both clusters are running", "leader".equals(targetCluster));
-
-        Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern");
-        request.setJsonEntity("{\"leader_index_patterns\": [\"logs-*\"], \"remote_cluster\": \"leader_cluster\"}");
-        assertOK(client().performRequest(request));
-
-        try (RestClient leaderClient = buildLeaderClient()) {
-            Settings settings = Settings.builder()
-                .put("index.soft_deletes.enabled", true)
-                .build();
-            request = new Request("PUT", "/logs-20190101");
-            request.setJsonEntity("{\"settings\": " + Strings.toString(settings) +
-                ", \"mappings\": {\"_doc\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}} }");
-            assertOK(leaderClient.performRequest(request));
-
-            for (int i = 0; i < 5; i++) {
-                String id = Integer.toString(i);
-                index(leaderClient, "logs-20190101", id, "field", i, "filtered_field", "true");
-            }
-        }
-
-        assertBusy(() -> {
-            Request statsRequest = new Request("GET", "/_ccr/stats");
-            Map<?, ?> response = toMap(client().performRequest(statsRequest));
-            response = (Map<?, ?>) response.get("auto_follow_stats");
-            assertThat(response.get("number_of_successful_follow_indices"), equalTo(1));
-
-            ensureYellow("logs-20190101");
-            verifyDocuments("logs-20190101", 5, "filtered_field:true");
-        });
-        assertBusy(() -> {
-            verifyCcrMonitoring("logs-20190101", "logs-20190101");
-            verifyAutoFollowMonitoring();
-        }, 30, TimeUnit.SECONDS);
-    }
-
 }