[7.x] Convert repository-* from integTest to [yaml | java]RestTest or…

… internalClusterTest (#60085) (#60404)

For OSS plugins that being with repository-*, integTest
task is now a no-op and all of the tests are now executed via a test,
yamlRestTest, javaRestTest, or internalClusterTest.

related: #56841
related: #59444

plugins/repository-azure/build.gradle

@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.test.InternalClusterTestPlugin
 
 import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
@@ -22,7 +23,8 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * specific language governing permissions and limitations
  * under the License.
  */
-apply plugin: 'elasticsearch.rest-resources'
+apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'The Azure Repository plugin adds support for Azure storage repositories.'
@@ -106,18 +108,18 @@ Map<String, Object> expansions = [
   'base_path': azureBasePath + "_integration_tests"
 ]
 
-processTestResources {
+processYamlRestTestResources {
   inputs.properties(expansions)
   MavenFilteringHack.filter(it, expansions)
 }
 
-test {
+internalClusterTest {
   // this is tested explicitly in a separate test task
   exclude '**/AzureStorageCleanupThirdPartyTests.class'
 }
 
 testClusters {
-  integTest {
+  yamlRestTest {
     keystore 'azure.client.integration_test.account', azureAccount
     if (azureKey != null && azureKey.isEmpty() == false) {
       keystore 'azure.client.integration_test.key', azureKey
@@ -134,7 +136,11 @@ testClusters {
 }
 
 task azureThirdPartyTest(type: Test) {
-  dependsOn tasks.integTest
+  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+  SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
+  setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
+  setClasspath(internalTestSourceSet.getRuntimeClasspath())
+  dependsOn tasks.internalClusterTest
   include '**/AzureStorageCleanupThirdPartyTests.class'
   systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
   systemProperty 'test.azure.key', azureKey ? azureKey : ""

plugins/repository-gcs/build.gradle

@@ -4,6 +4,8 @@ import java.security.KeyPairGenerator
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin
+import org.elasticsearch.gradle.test.InternalClusterTestPlugin
 
 import java.nio.file.Files
 import java.security.KeyPair
@@ -28,7 +30,8 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * specific language governing permissions and limitations
  * under the License.
  */
-apply plugin: 'elasticsearch.rest-resources'
+apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
@@ -254,12 +257,12 @@ Map<String, Object> expansions = [
   'base_path': gcsBasePath + "_integration_tests"
 ]
 
-processTestResources {
+processYamlRestTestResources {
   inputs.properties(expansions)
   MavenFilteringHack.filter(it, expansions)
 }
 
-test {
+internalClusterTest {
   // this is tested explicitly in a separate test task
   exclude '**/GoogleCloudStorageThirdPartyTests.class'
 }
@@ -276,33 +279,37 @@ final Closure testClustersConfiguration = {
   }
 }
 
-integTest {
+yamlRestTest {
   if (useFixture) {
     dependsOn createServiceAccountFile
   }
 }
-check.dependsOn integTest
 
 testClusters {
-  integTest testClustersConfiguration
+  all testClustersConfiguration
 }
 
 /*
  * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
  * an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
  */
-task largeBlobIntegTest(type: RestIntegTestTask) {
-  mustRunAfter integTest
+task largeBlobYamlRestTest(type: RestIntegTestTask) {
   dependsOn project(':plugins:repository-gcs').bundlePlugin
   if (useFixture) {
     dependsOn createServiceAccountFile
   }
+  runner {
+    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
+    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
+    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
+  }
 }
-check.dependsOn largeBlobIntegTest
 
-testClusters.largeBlobIntegTest testClustersConfiguration
+check.dependsOn largeBlobYamlRestTest
+
 testClusters {
-  largeBlobIntegTest {
+  largeBlobYamlRestTest {
     plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile
 
     // force large blob uploads by setting the threshold small, forcing this code path to be tested
@@ -311,7 +318,10 @@ testClusters {
 }
 
 task gcsThirdPartyTest(type: Test) {
-  dependsOn integTest,largeBlobIntegTest
+  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+  SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
+  setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
+  setClasspath(internalTestSourceSet.getRuntimeClasspath())
   include '**/GoogleCloudStorageThirdPartyTests.class'
   systemProperty 'tests.security.manager', false
   systemProperty 'test.google.bucket', gcsBucket

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java

@@ -26,6 +26,7 @@
 import com.sun.net.httpserver.HttpHandler;
 import fixture.gcs.FakeOAuth2HttpHandler;
 import org.apache.http.HttpStatus;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
@@ -94,9 +95,16 @@ private String httpServerUrl() {
         return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort();
     }
 
+    public static void assumeNotJava8() {
+        assumeFalse("This test is flaky on jdk8 - we suspect a JDK bug to trigger some assertion in the HttpServer implementation used " +
+            "to emulate the server side logic of Google Cloud Storage. See https://bugs.openjdk.java.net/browse/JDK-8180754, " +
+            "https://github.com/elastic/elasticsearch/pull/51933 and https://github.com/elastic/elasticsearch/issues/52906 " +
+            "for more background on this issue.", JavaVersion.current().equals(JavaVersion.parse("8")));
+    }
+
     @BeforeClass
     public static void skipJava8() {
-        GoogleCloudStorageBlobStoreRepositoryTests.assumeNotJava8();
+        assumeNotJava8();
     }
 
     @Override

plugins/repository-s3/build.gradle

@@ -1,6 +1,8 @@
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin
+import org.elasticsearch.gradle.test.InternalClusterTestPlugin
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
@@ -22,7 +24,8 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * specific language governing permissions and limitations
  * under the License.
  */
-apply plugin: 'elasticsearch.rest-resources'
+apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'The S3 repository plugin adds S3 repositories'
@@ -55,6 +58,12 @@ dependencies {
   testImplementation project(':test:fixtures:s3-fixture')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
+  }
+}
+
 tasks.named("dependencyLicenses").configure {
   mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
   mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
@@ -147,7 +156,7 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
 }
 
-processTestResources {
+processYamlRestTestResources {
   Map<String, Object> expansions = [
     'permanent_bucket': s3PermanentBucket,
     'permanent_base_path': s3PermanentBasePath + "_integration_tests",
@@ -163,13 +172,12 @@ processTestResources {
   MavenFilteringHack.filter(it, expansions)
 }
 
-test {
+internalClusterTest {
   // this is tested explicitly in a separate test task
   exclude '**/S3RepositoryThirdPartyTests.class'
 }
 
-// IntegTest
-integTest {
+yamlRestTest {
   runner {
     systemProperty 'tests.rest.blacklist', (
       useFixture ?
@@ -184,7 +192,7 @@ integTest {
   }
 }
 
-testClusters.integTest {
+testClusters.yamlRestTest {
   keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
   keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
 
@@ -219,10 +227,15 @@ testClusters.integTest {
 if (useFixture) {
   testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
 
-  task integTestMinio(type: RestIntegTestTask) {
+  task yamlRestTestMinio(type: RestIntegTestTask) {
     description = "Runs REST tests using the Minio repository."
     dependsOn tasks.bundlePlugin
     runner {
+      SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+      SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
+      setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
+      setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
+
       // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
       systemProperty 'tests.rest.blacklist', [
         'repository_s3/30_repository_temporary_credentials/*',
@@ -231,9 +244,9 @@ if (useFixture) {
       ].join(",")
     }
   }
-  check.dependsOn(integTestMinio)
+  check.dependsOn(yamlRestTestMinio)
 
-  testClusters.integTestMinio {
+  testClusters.yamlRestTestMinio {
     keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
     keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
     setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE
@@ -244,11 +257,14 @@ if (useFixture) {
 // ECS
 if (useFixture) {
   testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs')
-
-  task integTestECS(type: RestIntegTestTask.class) {
+  task yamlRestTestECS(type: RestIntegTestTask.class) {
     description = "Runs tests using the ECS repository."
     dependsOn('bundlePlugin')
     runner {
+      SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+      SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
+      setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
+      setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
       systemProperty 'tests.rest.blacklist', [
         'repository_s3/10_basic/*',
         'repository_s3/20_repository_permanent_credentials/*',
@@ -257,9 +273,9 @@ if (useFixture) {
       ].join(",")
     }
   }
-  check.dependsOn(integTestECS)
+  check.dependsOn(yamlRestTestECS)
 
-  testClusters.integTestECS {
+  testClusters.yamlRestTestECS {
     setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE
     plugin tasks.bundlePlugin.archiveFile
     environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE
@@ -268,16 +284,17 @@ if (useFixture) {
 
 // 3rd Party Tests
 task s3ThirdPartyTest(type: Test) {
+  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+  SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
+  setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
+  setClasspath(internalTestSourceSet.getRuntimeClasspath())
   include '**/S3RepositoryThirdPartyTests.class'
   systemProperty 'test.s3.account', s3PermanentAccessKey
   systemProperty 'test.s3.key', s3PermanentSecretKey
   systemProperty 'test.s3.bucket', s3PermanentBucket
   nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
   if (useFixture) {
-    dependsOn tasks.integTestMinio
     nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }"
-  } else {
-    dependsOn tasks.integTest
   }
 }
 check.dependsOn(s3ThirdPartyTest)

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java → plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -304,3 +304,4 @@ private boolean isMultiPartUpload(String request) {
         }
     }
 }
+