Skip to content

Commit

Permalink
Convert second 1/2 x-pack plugins from integTest to [yaml | java]Rest…
Browse files Browse the repository at this point in the history
…Test or internalClusterTest (elastic#61802)

For 1/2 the plugins in x-pack, the integTest
task is now a no-op and all of the tests are now executed via a test,
yamlRestTest, javaRestTest, or internalClusterTest.

This includes the following projects:
security, spatial, stack, transform, vecotrs, voting-only-node, and watcher.

A few of the more specialized qa projects within these plugins
have not been changed with this PR due to additional complexity which should
be addressed separately.

related: elastic#60630
related: elastic#56841
related: elastic#59939
related: elastic#55896
# Conflicts:
#	x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java
#	x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java
#	x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
#	x-pack/plugin/transform/qa/multi-node-tests/build.gradle
#	x-pack/plugin/transform/qa/single-node-tests/build.gradle
#	x-pack/plugin/watcher/build.gradle
  • Loading branch information
jakelandis committed Sep 2, 2020
1 parent 07ab0be commit b14be75
Show file tree
Hide file tree
Showing 242 changed files with 188 additions and 173 deletions.
4 changes: 3 additions & 1 deletion x-pack/plugin/searchable-snapshots/build.gradle
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import org.elasticsearch.gradle.info.BuildParams

apply plugin: 'elasticsearch.internal-cluster-test'
apply plugin: 'elasticsearch.esplugin'
esplugin {
name 'searchable-snapshots'
Expand All @@ -10,7 +12,7 @@ archivesBaseName = 'x-pack-searchable-snapshots'

dependencies {
compileOnly project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
internalClusterTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}

// xpack modules are installed in real clusters as the meta plugin, so
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsRequest;
import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsResponse;
import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
import org.hamcrest.Matchers;

import java.io.IOException;
import java.nio.file.Files;
Expand Down Expand Up @@ -78,7 +79,6 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.lessThanOrEqualTo;

public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegTestCase {
Expand Down Expand Up @@ -737,7 +737,7 @@ private void assertSearchableSnapshotStats(String indexName, boolean cacheEnable
new SearchableSnapshotsStatsRequest(indexName)
).actionGet();
final NumShards restoredNumShards = getNumShards(indexName);
assertThat(statsResponse.getStats(), hasSize(restoredNumShards.totalNumShards));
assertThat(statsResponse.getStats(), Matchers.hasSize(restoredNumShards.totalNumShards));

final long totalSize = statsResponse.getStats()
.stream()
Expand Down
10 changes: 9 additions & 1 deletion x-pack/plugin/security/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import org.elasticsearch.gradle.info.BuildParams

apply plugin: 'elasticsearch.esplugin'
apply plugin: 'elasticsearch.publish'
apply plugin: 'elasticsearch.internal-cluster-test'
esplugin {
name 'x-pack-security'
description 'Elasticsearch Expanded Pack Plugin - Security'
Expand All @@ -21,6 +22,7 @@ dependencies {
testImplementation project(path: xpackModule('sql:sql-action'))

testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
internalClusterTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')

api 'com.unboundid:unboundid-ldapsdk:4.0.8'

Expand Down Expand Up @@ -134,11 +136,17 @@ dependencies {
compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"

processInternalClusterTestResources {
from(project(xpackModule('core')).file('src/main/config'))
from(project(xpackModule('core')).file('src/test/resources'))
}

processTestResources {
from(project(xpackModule('core')).file('src/main/config'))
from(project(xpackModule('core')).file('src/test/resources'))
}


configurations {
testArtifacts.extendsFrom testRuntime
testArtifacts.extendsFrom testImplementation
Expand Down Expand Up @@ -464,7 +472,7 @@ if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
)
}

test {
internalClusterTest {
/*
* We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
* other if we allow them to set the number of available processors as it's set-once in Netty.
Expand Down
40 changes: 20 additions & 20 deletions x-pack/plugin/security/qa/basic-enable-security/build.gradle
Original file line number Diff line number Diff line change
@@ -1,36 +1,36 @@
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.elasticsearch.gradle.test.rest.JavaRestTestPlugin

apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.java-rest-test'

dependencies {
testImplementation project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}

integTest {
javaRestTest {
description = "Run tests against a cluster that doesn't have security"
systemProperty 'tests.has_security', 'false'
}

testClusters.integTest {
testClusters.javaRestTest {
testDistribution = 'DEFAULT'
numberOfNodes = 2
setting 'xpack.ml.enabled', 'false'
setting 'xpack.license.self_generated.type', 'basic'
setting 'xpack.security.enabled', 'false'
}

task integTestSecurity(type: StandaloneRestIntegTestTask) {
description = "Run tests against a cluster that has security"
useCluster testClusters.integTest
dependsOn integTest
task javaRestTestWithSecurity(type: StandaloneRestIntegTestTask) {
description = "Run tests against a cluster that has security enabled"
useCluster testClusters.javaRestTest
dependsOn javaRestTest
systemProperty 'tests.has_security', 'true'

testClassesDirs = sourceSets.javaRestTest.output.classesDirs
classpath = sourceSets.javaRestTest.runtimeClasspath
doFirst {
testClusters.integTest {
testClusters.javaRestTest {
// Reconfigure cluster to enable security
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.authc.anonymous.roles', 'anonymous'
Expand All @@ -40,18 +40,18 @@ task integTestSecurity(type: StandaloneRestIntegTestTask) {
setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
setting 'xpack.security.transport.ssl.certificate_authorities', 'ca.crt'

extraConfigFile 'transport.key', file('src/test/resources/ssl/transport.key')
extraConfigFile 'transport.crt', file('src/test/resources/ssl/transport.crt')
extraConfigFile 'ca.crt', file('src/test/resources/ssl/ca.crt')
extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
extraConfigFile 'transport.key', file('src/javaRestTest/resources/ssl/transport.key')
extraConfigFile 'transport.crt', file('src/javaRestTest/resources/ssl/transport.crt')
extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt')
extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')

user username: "admin_user", password: "admin-password"
user username: "security_test_user", password: "security-test-password", role: "security_test_role"

restart()
}
nonInputProperties.systemProperty 'tests.rest.cluster', "${-> testClusters.integTest.getAllHttpSocketURI().join(",")}"
nonInputProperties.systemProperty 'tests.rest.cluster', "${-> testClusters.javaRestTest.getAllHttpSocketURI().join(",")}"
}
}
tasks.named("check").configure { dependsOn(integTestSecurity) }
tasks.named("check").configure { dependsOn(javaRestTestWithSecurity) }

15 changes: 7 additions & 8 deletions x-pack/plugin/security/qa/security-basic/build.gradle
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'

apply plugin: 'elasticsearch.java-rest-test'

dependencies {
testImplementation project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}

testClusters.integTest {
testClusters.all {
testDistribution = 'DEFAULT'
numberOfNodes = 2

Expand All @@ -21,7 +20,7 @@ testClusters.integTest {
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.authc.api_key.enabled', 'true'

extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
user username: "admin_user", password: "admin-password"
user username: "security_test_user", password: "security-test-password", role: "security_test_role"
}
12 changes: 5 additions & 7 deletions x-pack/plugin/security/qa/security-disabled/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,15 @@
* defined, it should be not fail
*/

apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.java-rest-test'

dependencies {
testImplementation project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}

testClusters.integTest {
testClusters.all {
testDistribution = 'DEFAULT'
numberOfNodes = 2

Expand Down
12 changes: 5 additions & 7 deletions x-pack/plugin/security/qa/security-not-enabled/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,15 @@
* defined, it should be not fail
*/

apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.java-rest-test'

dependencies {
testImplementation project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}

testClusters.integTest {
testClusters.all {
testDistribution = 'DEFAULT'
numberOfNodes = 2

Expand Down
14 changes: 6 additions & 8 deletions x-pack/plugin/security/qa/security-trial/build.gradle
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.java-rest-test'

dependencies {
testImplementation project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}

testClusters.integTest {
testClusters.javaRestTest {
testDistribution = 'DEFAULT'
numberOfNodes = 2

Expand All @@ -21,7 +19,7 @@ testClusters.integTest {
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.authc.api_key.enabled', 'true'

extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
user username: "admin_user", password: "admin-password"
user username: "security_test_user", password: "security-test-password", role: "security_test_role"
}
22 changes: 7 additions & 15 deletions x-pack/plugin/security/qa/tls-basic/build.gradle
Original file line number Diff line number Diff line change
@@ -1,28 +1,20 @@
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.java-rest-test'

dependencies {
testImplementation project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}

forbiddenPatterns {
exclude '**/*.key'
exclude '**/*.p12'
}


testClusters.integTest {
testClusters.javaRestTest {
testDistribution = 'DEFAULT'
numberOfNodes = 2

extraConfigFile 'http.key', file('src/test/resources/ssl/http.key')
extraConfigFile 'http.crt', file('src/test/resources/ssl/http.crt')
extraConfigFile 'transport.key', file('src/test/resources/ssl/transport.key')
extraConfigFile 'transport.crt', file('src/test/resources/ssl/transport.crt')
extraConfigFile 'ca.crt', file('src/test/resources/ssl/ca.crt')
extraConfigFile 'http.key', file('src/javaRestTest/resources/ssl/http.key')
extraConfigFile 'http.crt', file('src/javaRestTest/resources/ssl/http.crt')
extraConfigFile 'transport.key', file('src/javaRestTest/resources/ssl/transport.key')
extraConfigFile 'transport.crt', file('src/javaRestTest/resources/ssl/transport.crt')
extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt')

setting 'xpack.ml.enabled', 'false'
setting 'xpack.license.self_generated.type', 'basic'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ public void testGetMappings() throws Exception {
MappingMetadata mappingMetadata = mappingsMap.get(index).get(type);
assertThat(mappingMetadata.getSourceAsMap(), hasKey("properties"));
assertThat(mappingMetadata.getSourceAsMap().get("properties"), instanceOf(Map.class));
@SuppressWarnings("unchecked")
Map<String, Object> propertiesMap = (Map<String, Object>) mappingMetadata.getSourceAsMap().get("properties");
assertThat(propertiesMap, hasKey(field));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ protected Settings transportClientSettings() {

public void testSecurityServerTransportServiceWrapsAllHandlers() {
for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
@SuppressWarnings("rawtypes")
RequestHandlerRegistry handler = transportService.transport.getRequestHandlers()
.getHandler(TransportService.HANDSHAKE_ACTION_NAME);
assertEquals(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,7 @@ public void testGetApiKeysForApiKeyName() throws InterruptedException, Execution
Client client = client().filterWithHeader(headers);
SecurityClient securityClient = new SecurityClient(client);
PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
@SuppressWarnings("unchecked")
List<CreateApiKeyResponse> responses = randomFrom(createApiKeyResponses1, createApiKeyResponses2);
securityClient.getApiKey(GetApiKeyRequest.usingApiKeyName(responses.get(0).getName(), false), listener);
verifyGetResponse(1, responses, listener.get(), Collections.singleton(responses.get(0).getId()), null);
Expand Down Expand Up @@ -651,6 +652,7 @@ public void testGetApiKeysOwnedByRunAsUserWillNotWorkWhenAuthUserInfoIsGiven() t
final List<CreateApiKeyResponse> userWithManageOwnApiKeyRoleApiKeys = createApiKeys("user_with_manage_own_api_key_role",
"user_with_run_as_role", noOfApiKeysForUserWithManageApiKeyRole, null, "monitor");
PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
@SuppressWarnings("unchecked")
final Tuple<String,String> invalidRealmAndUserPair = randomFrom(
new Tuple<>("file", "user_with_run_as_role"),
new Tuple<>("index", "user_with_manage_own_api_key_role"),
Expand Down Expand Up @@ -761,6 +763,7 @@ public void testInvalidateApiKeysOwnedByRunAsUserWillNotWorkWhenAuthUserInfoIsGi
List<CreateApiKeyResponse> userWithManageApiKeyRoleApiKeys = createApiKeys("user_with_manage_own_api_key_role",
"user_with_run_as_role", noOfApiKeysForUserWithManageApiKeyRole, null, "monitor");
PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
@SuppressWarnings("unchecked")
final Tuple<String,String> invalidRealmAndUserPair = randomFrom(
new Tuple<>("file", "user_with_run_as_role"),
new Tuple<>("index", "user_with_manage_own_api_key_role"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,7 @@ public void testRefreshingMultipleTimesFails() throws Exception {
.setFetchSource("refresh_token", Strings.EMPTY_STRING)
.get();
assertNotNull(updateResponse);
@SuppressWarnings("unchecked")
Map<String, Object> refreshTokenMap = (Map<String, Object>) updateResponse.getGetResult().sourceAsMap().get("refresh_token");
assertTrue(
Instant.ofEpochMilli((long) refreshTokenMap.get("refresh_time")).isBefore(Instant.now().minus(30L, ChronoUnit.SECONDS)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,7 @@ public void testRolesUsageStats() throws Exception {
assertThat(usage.get("dls"), is(dls));
}

@SuppressWarnings("unchecked")
public void testRealmUsageStats() {
final int numNativeUsers = scaledRandomIntBetween(1, 32);
SecurityClient securityClient = new SecurityClient(client());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.elasticsearch.test.SecurityIntegTestCase;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.xpack.core.common.socket.SocketAccess;
import org.elasticsearch.xpack.security.transport.filter.IPFilter;
import org.junit.BeforeClass;

import java.io.IOException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.SecurityIntegTestCase;
import org.elasticsearch.xpack.security.transport.filter.IPFilter;
import org.junit.BeforeClass;

import java.net.InetAddress;
Expand Down
6 changes: 4 additions & 2 deletions x-pack/plugin/spatial/build.gradle
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apply plugin: 'elasticsearch.esplugin'
apply plugin: 'elasticsearch.rest-resources'
apply plugin: 'elasticsearch.internal-cluster-test'
apply plugin: 'elasticsearch.yaml-rest-test'

esplugin {
name 'spatial'
Expand All @@ -11,6 +12,7 @@ esplugin {
dependencies {
compileOnly project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
yamlRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
api project(path: ':modules:geo', configuration: 'default')
restTestConfig project(path: ':modules:geo', configuration: 'restTests')
}
Expand All @@ -24,7 +26,7 @@ restResources {
}
}

testClusters.integTest {
testClusters.all {
setting 'xpack.license.self_generated.type', 'trial'
setting 'indices.breaker.request.limit', '25kb'
testDistribution = 'DEFAULT'
Expand Down
Loading

0 comments on commit b14be75

Please sign in to comment.