From 4520e88b2257ec7fc8b8cdeca29cb68d28df0b4b Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 23 May 2019 10:16:48 -0700 Subject: [PATCH] Remove transport client from xpack (#42202) This commit removes support for the transport client from xpack. --- build.gradle | 3 +- .../gradle/test/RestIntegTestTask.groovy | 11 +- .../build.gradle | 2 +- .../example/CustomAuthorizationEngineIT.java | 70 +++--- .../java/org/elasticsearch/node/Node.java | 2 + .../java/org/elasticsearch/node/MockNode.java | 5 + .../elasticsearch/test/ESIntegTestCase.java | 16 +- .../test/ExternalTestCluster.java | 57 +++-- .../ccs-clients-integrations/java.asciidoc | 203 ----------------- x-pack/docs/en/watcher/java.asciidoc | 130 ----------- .../java/org/elasticsearch/xpack/ccr/Ccr.java | 6 - .../xpack/core/XPackClientPlugin.java | 53 +---- .../elasticsearch/xpack/core/XPackPlugin.java | 16 +- .../AbstractLicensesIntegrationTestCase.java | 1 + .../license/StartBasicLicenseTests.java | 2 +- .../license/StartTrialLicenseTests.java | 2 +- .../snapshots/SourceOnlySnapshotIT.java | 2 +- .../integration/DataFrameIntegTestCase.java | 210 +++++++++--------- .../integration/DataFrameTransformIT.java | 24 +- .../DataFrameTransformProgressIT.java | 83 +++---- .../xpack/dataframe/DataFrame.java | 14 +- .../transforms/TransformProgressGatherer.java | 27 ++- .../xpack/indexlifecycle/IndexLifecycle.java | 10 +- .../IndexLifecycleInitialisationTests.java | 20 +- .../xpack/logstash/Logstash.java | 6 - .../ml/integration/MlNativeIntegTestCase.java | 46 +++- .../xpack/ml/MachineLearning.java | 18 +- .../xpack/ml/MachineLearningFeatureSet.java | 3 +- .../xpack/monitoring/Monitoring.java | 8 +- .../MonitoringPluginClientTests.java | 13 -- .../elasticsearch/xpack/rollup/Rollup.java | 10 +- .../xpack/security/Security.java | 52 ++--- .../support/AbstractSecurityModule.java | 47 ---- .../integration/BulkUpdateTests.java | 10 +- .../integration/ClearRolesCacheTests.java | 2 +- .../integration/FieldLevelSecurityTests.java | 2 +- .../MultipleIndicesPermissionsTests.java | 4 +- .../PermissionPrecedenceTests.java | 16 +- .../integration/SecurityClearScrollTests.java | 4 +- .../ShrinkIndexWithSecurityTests.java | 2 +- .../test/NativeRealmIntegTestCase.java | 2 +- .../test/SecurityIntegTestCase.java | 53 +---- .../xpack/security/SecurityTests.java | 12 +- .../xpack/security/TemplateUpgraderTests.java | 2 +- .../AuditTrailSettingsUpdateTests.java | 2 +- .../security/authz/SecurityScrollTests.java | 11 - .../filter/IpFilteringIntegrationTests.java | 6 - .../filter/IpFilteringUpdateTests.java | 2 +- .../netty4/IPHostnameVerificationTests.java | 88 -------- .../xpack/ssl/SSLClientAuthTests.java | 2 +- .../xpack/ssl/SSLTrustRestrictionsTests.java | 11 +- .../elasticsearch/xpack/watcher/Watcher.java | 10 +- .../xpack/security/ReindexWithSecurityIT.java | 188 +++++++++++----- x-pack/qa/security-client-tests/build.gradle | 40 ---- .../qa/SecurityTransportClientIT.java | 125 ----------- .../build.gradle | 2 +- .../example/realm/CustomRealmIT.java | 108 ++------- .../example/role/CustomRolesProviderIT.java | 69 +++--- .../smoketest/PreventFailingBuildIT.java | 4 +- x-pack/qa/smoke-test-plugins-ssl/build.gradle | 1 + .../SmokeTestMonitoringWithSecurityIT.java | 171 +++++++++----- x-pack/qa/transport-client-tests/build.gradle | 22 -- .../ml/client/ESXPackSmokeClientTestCase.java | 153 ------------- .../xpack/ml/client/MLTransportClientIT.java | 179 --------------- x-pack/transport-client/build.gradle | 41 ---- .../client/PreBuiltXPackTransportClient.java | 66 ------ .../PreBuiltXPackTransportClientTests.java | 30 --- 67 files changed, 699 insertions(+), 1913 deletions(-) delete mode 100644 x-pack/docs/en/security/ccs-clients-integrations/java.asciidoc delete mode 100644 x-pack/docs/en/watcher/java.asciidoc delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/AbstractSecurityModule.java delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java delete mode 100644 x-pack/qa/security-client-tests/build.gradle delete mode 100644 x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java delete mode 100644 x-pack/qa/transport-client-tests/build.gradle delete mode 100644 x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java delete mode 100644 x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java delete mode 100644 x-pack/transport-client/build.gradle delete mode 100644 x-pack/transport-client/src/main/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClient.java delete mode 100644 x-pack/transport-client/src/test/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClientTests.java diff --git a/build.gradle b/build.gradle index 8794a1f930523..7de02b814da86 100644 --- a/build.gradle +++ b/build.gradle @@ -241,8 +241,7 @@ allprojects { "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator', "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval', // for security example plugins - "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core', - "org.elasticsearch.client:x-pack-transport:${version}": ':x-pack:transport-client' + "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core' ] /* diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index ef784b6f901d1..52c498aa98d79 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -114,13 +114,14 @@ class RestIntegTestTask extends DefaultTask { runner.ext.nonInputProperties = nonInputProperties if (System.getProperty("tests.rest.cluster") == null) { - if (System.getProperty("tests.cluster") != null) { - throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null") + if (System.getProperty("tests.cluster") != null || System.getProperty("tests.clustername") != null) { + throw new IllegalArgumentException("tests.rest.cluster, tests.cluster, and tests.clustername must all be null or non-null") } if (usesTestclusters == true) { ElasticsearchCluster cluster = project.testClusters."${name}" nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }") nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }") + nonInputProperties.systemProperty('tests.clustername', "${-> cluster.getName() }") } else { // we pass all nodes to the rest cluster to allow the clients to round-robin between them // this is more realistic than just talking to a single node @@ -130,6 +131,7 @@ class RestIntegTestTask extends DefaultTask { // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass // both as separate sysprops nonInputProperties.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}") + nonInputProperties.systemProperty('tests.clustername', "${-> nodes[0].clusterName}") // dump errors and warnings from cluster log on failure TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() { @@ -150,12 +152,13 @@ class RestIntegTestTask extends DefaultTask { } } } else { - if (System.getProperty("tests.cluster") == null) { - throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null") + if (System.getProperty("tests.cluster") == null || System.getProperty("tests.clustername") == null) { + throw new IllegalArgumentException("tests.rest.cluster, tests.cluster, and tests.clustername must all be null or non-null") } // an external cluster was specified and all responsibility for cluster configuration is taken by the user runner.systemProperty('tests.rest.cluster', System.getProperty("tests.rest.cluster")) runner.systemProperty('test.cluster', System.getProperty("tests.cluster")) + runner.systemProperty('test.clustername', System.getProperty("tests.clustername")) } // copy the rest spec/tests into the test resources diff --git a/plugins/examples/security-authorization-engine/build.gradle b/plugins/examples/security-authorization-engine/build.gradle index fba9580525bcc..787cc230eeb18 100644 --- a/plugins/examples/security-authorization-engine/build.gradle +++ b/plugins/examples/security-authorization-engine/build.gradle @@ -12,7 +12,7 @@ esplugin { dependencies { compileOnly "org.elasticsearch.plugin:x-pack-core:${versions.elasticsearch}" - testCompile "org.elasticsearch.client:x-pack-transport:${versions.elasticsearch}" + testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${versions.elasticsearch}" } integTest { diff --git a/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java b/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java index 9daf9bd01a8bc..4342b2a4b88f0 100644 --- a/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java +++ b/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java @@ -24,22 +24,21 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.security.PutUserRequest; +import org.elasticsearch.client.security.RefreshPolicy; +import org.elasticsearch.client.security.user.User; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.XPackClientPlugin; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.client.SecurityClient; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; -import java.util.Collection; import java.util.Collections; +import java.util.List; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; @@ -50,26 +49,21 @@ * an external cluster with the custom authorization plugin installed to validate the functionality * when running as a plugin */ -public class CustomAuthorizationEngineIT extends ESIntegTestCase { +public class CustomAuthorizationEngineIT extends ESRestTestCase { @Override - protected Settings externalClusterClientSettings() { + protected Settings restClientSettings() { final String token = "Basic " + Base64.getEncoder().encodeToString(("test_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8)); return Settings.builder() .put(ThreadContext.PREFIX + ".Authorization", token) - .put(NetworkModule.TRANSPORT_TYPE_KEY, "security4") .build(); } - @Override - protected Collection> transportClientPlugins() { - return Collections.singleton(XPackClientPlugin.class); - } - public void testClusterAction() throws IOException { - SecurityClient securityClient = new SecurityClient(client()); - securityClient.preparePutUser("custom_user", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get(); + RestHighLevelClient restClient = new TestRestHighLevelClient(); + restClient.security().putUser(PutUserRequest.withPassword(new User("custom_user", List.of("custom_superuser")), + "x-pack-test-password".toCharArray(), true, RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); { RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); @@ -77,25 +71,27 @@ public void testClusterAction() throws IOException { basicAuthHeaderValue("custom_user", new SecureString("x-pack-test-password".toCharArray()))); Request request = new Request("GET", "_cluster/health"); request.setOptions(options); - Response response = getRestClient().performRequest(request); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } { - securityClient.preparePutUser("custom_user2", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "not_superuser").get(); + restClient.security().putUser(PutUserRequest.withPassword(new User("custom_user2", List.of("not_superuser")), + "x-pack-test-password".toCharArray(), true, RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue("custom_user2", new SecureString("x-pack-test-password".toCharArray()))); Request request = new Request("GET", "_cluster/health"); request.setOptions(options); - ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); } } public void testIndexAction() throws IOException { - SecurityClient securityClient = new SecurityClient(client()); - securityClient.preparePutUser("custom_user", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get(); + RestHighLevelClient restClient = new TestRestHighLevelClient(); + restClient.security().putUser(PutUserRequest.withPassword(new User("custom_user", List.of("custom_superuser")), + "x-pack-test-password".toCharArray(), true, RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); { RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); @@ -103,27 +99,31 @@ public void testIndexAction() throws IOException { basicAuthHeaderValue("custom_user", new SecureString("x-pack-test-password".toCharArray()))); Request request = new Request("PUT", "/index"); request.setOptions(options); - Response response = getRestClient().performRequest(request); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } { - securityClient.preparePutUser("custom_user2", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "not_superuser").get(); + restClient.security().putUser(PutUserRequest.withPassword(new User("custom_user2", List.of("not_superuser")), + "x-pack-test-password".toCharArray(), true, RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue("custom_user2", new SecureString("x-pack-test-password".toCharArray()))); Request request = new Request("PUT", "/index"); request.setOptions(options); - ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); } } public void testRunAs() throws IOException { - SecurityClient securityClient = new SecurityClient(client()); - securityClient.preparePutUser("custom_user", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get(); - securityClient.preparePutUser("custom_user2", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get(); - securityClient.preparePutUser("custom_user3", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "not_superuser").get(); + RestHighLevelClient restClient = new TestRestHighLevelClient(); + restClient.security().putUser(PutUserRequest.withPassword(new User("custom_user", List.of("custom_superuser")), + "x-pack-test-password".toCharArray(), true, RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); + restClient.security().putUser(PutUserRequest.withPassword(new User("custom_user2", List.of("custom_superuser")), + "x-pack-test-password".toCharArray(), true, RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); + restClient.security().putUser(PutUserRequest.withPassword(new User("custom_user3", List.of("not_superuser")), + "x-pack-test-password".toCharArray(), true, RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); { RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); @@ -132,7 +132,7 @@ public void testRunAs() throws IOException { options.addHeader("es-security-runas-user", "custom_user2"); Request request = new Request("GET", "/_security/_authenticate"); request.setOptions(options); - Response response = getRestClient().performRequest(request); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); String responseStr = EntityUtils.toString(response.getEntity()); assertThat(responseStr, containsString("custom_user2")); @@ -145,7 +145,7 @@ public void testRunAs() throws IOException { options.addHeader("es-security-runas-user", "custom_user3"); Request request = new Request("PUT", "/index"); request.setOptions(options); - ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); } @@ -156,8 +156,14 @@ public void testRunAs() throws IOException { options.addHeader("es-security-runas-user", "custom_user2"); Request request = new Request("PUT", "/index"); request.setOptions(options); - ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); } } + + private class TestRestHighLevelClient extends RestHighLevelClient { + TestRestHighLevelClient() { + super(client(), restClient -> {}, Collections.emptyList()); + } + } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index d338a4663a32a..fab08ab1c03f7 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -251,6 +251,7 @@ public class Node implements Closeable { private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; + final NamedWriteableRegistry namedWriteableRegistry; public Node(Environment environment) { this(environment, Collections.emptyList(), true); @@ -589,6 +590,7 @@ protected Node( this.pluginLifecycleComponents = Collections.unmodifiableList(pluginLifecycleComponents); client.initialize(injector.getInstance(new Key>() {}), () -> clusterService.localNode().getId(), transportService.getRemoteClusterService()); + this.namedWriteableRegistry = namedWriteableRegistry; logger.debug("initializing HTTP handlers ..."); actionModule.initRestHandlers(() -> clusterService.state().nodes()); diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index 31b8ba01dc4a8..b43e438bc3210 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.MockInternalClusterInfoService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -179,4 +180,8 @@ protected HttpServerTransport newHttpTransport(NetworkModule networkModule) { protected void configureNodeAndClusterIdStateListener(ClusterService clusterService) { //do not configure this in tests as this is causing SetOnce to throw exceptions when jvm is used for multiple tests } + + public NamedWriteableRegistry getNamedWriteableRegistry() { + return namedWriteableRegistry; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index d45c83444b2fc..94a8e9b7728ce 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -289,6 +289,11 @@ public abstract class ESIntegTestCase extends ESTestCase { */ public static final String TESTS_CLUSTER = "tests.cluster"; + /** + * Key used to eventually switch to using an external cluster and provide the cluster name + */ + public static final String TESTS_CLUSTER_NAME = "tests.clustername"; + /** * Key used to retrieve the index random seed from the index settings on a running node. * The value of this seed can be used to initialize a random context for a specific index. @@ -1829,7 +1834,7 @@ protected Settings transportClientSettings() { return Settings.EMPTY; } - private ExternalTestCluster buildExternalCluster(String clusterAddresses) throws IOException { + private ExternalTestCluster buildExternalCluster(String clusterAddresses, String clusterName) throws IOException { String[] stringAddresses = clusterAddresses.split(","); TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; int i = 0; @@ -1838,7 +1843,8 @@ private ExternalTestCluster buildExternalCluster(String clusterAddresses) throws InetAddress inetAddress = InetAddress.getByName(url.getHost()); transportAddresses[i++] = new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); } - return new ExternalTestCluster(createTempDir(), externalClusterClientSettings(), transportClientPlugins(), transportAddresses); + return new ExternalTestCluster(createTempDir(), externalClusterClientSettings(), nodePlugins(), getClientWrapper(), clusterName, + transportAddresses); } protected Settings externalClusterClientSettings() { @@ -1855,7 +1861,11 @@ protected TestCluster buildTestCluster(Scope scope, long seed) throws IOExceptio if (scope == Scope.TEST) { throw new IllegalArgumentException("Cannot run TEST scope test with " + TESTS_CLUSTER); } - return buildExternalCluster(clusterAddresses); + String clusterName = System.getProperty(TESTS_CLUSTER_NAME); + if (Strings.isNullOrEmpty(clusterName)) { + throw new IllegalArgumentException("External test cluster name must be provided"); + } + return buildExternalCluster(clusterAddresses, clusterName); } final String nodePrefix; diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java index 74edfd3a46514..e77d143e50d99 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; @@ -32,18 +33,23 @@ import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.node.MockNode; +import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.MockTransportClient; import org.elasticsearch.transport.nio.MockNioTransportPlugin; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; -import java.util.Collections; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.elasticsearch.test.ESTestCase.getTestTransportType; import static org.hamcrest.Matchers.equalTo; @@ -61,7 +67,8 @@ public final class ExternalTestCluster extends TestCluster { private static final AtomicInteger counter = new AtomicInteger(); public static final String EXTERNAL_CLUSTER_PREFIX = "external_"; - private final MockTransportClient client; + private final MockNode node; + private final Client client; private final InetSocketAddress[] httpAddresses; @@ -71,13 +78,21 @@ public final class ExternalTestCluster extends TestCluster { private final int numMasterAndDataNodes; public ExternalTestCluster(Path tempDir, Settings additionalSettings, Collection> pluginClasses, - TransportAddress... transportAddresses) { + Function clientWrapper, String clusterName, TransportAddress... transportAddresses) { super(0); + this.clusterName = clusterName; Settings.Builder clientSettingsBuilder = Settings.builder() .put(additionalSettings) - .put("node.name", InternalTestCluster.TRANSPORT_CLIENT_PREFIX + EXTERNAL_CLUSTER_PREFIX + counter.getAndIncrement()) - .put("client.transport.ignore_cluster_name", true) - .put(Environment.PATH_HOME_SETTING.getKey(), tempDir); + .put("node.master", false) + .put("node.data", false) + .put("node.ingest", false) + .put("node.name", EXTERNAL_CLUSTER_PREFIX + counter.getAndIncrement()) + .put("cluster.name", clusterName) + .putList("discovery.seed_hosts", + Arrays.stream(transportAddresses).map(TransportAddress::toString).collect(Collectors.toList())); + if (Environment.PATH_HOME_SETTING.exists(additionalSettings) == false) { + clientSettingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), tempDir); + } boolean addMockTcpTransport = additionalSettings.get(NetworkModule.TRANSPORT_TYPE_KEY) == null; if (addMockTcpTransport) { @@ -88,13 +103,15 @@ public ExternalTestCluster(Path tempDir, Settings additionalSettings, Collection pluginClasses.add(MockNioTransportPlugin.class); } } + pluginClasses = new ArrayList<>(pluginClasses); + pluginClasses.add(MockHttpTransport.TestPlugin.class); Settings clientSettings = clientSettingsBuilder.build(); - MockTransportClient client = new MockTransportClient(clientSettings, pluginClasses); + MockNode node = new MockNode(clientSettings, pluginClasses); + Client client = clientWrapper.apply(node.client()); try { - client.addTransportAddresses(transportAddresses); + node.start(); NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear().setSettings(true).setHttp(true).get(); httpAddresses = new InetSocketAddress[nodeInfos.getNodes().size()]; - this.clusterName = nodeInfos.getClusterName().value(); int dataNodes = 0; int masterAndDataNodes = 0; for (int i = 0; i < nodeInfos.getNodes().size(); i++) { @@ -110,10 +127,22 @@ public ExternalTestCluster(Path tempDir, Settings additionalSettings, Collection this.numDataNodes = dataNodes; this.numMasterAndDataNodes = masterAndDataNodes; this.client = client; + this.node = node; logger.info("Setup ExternalTestCluster [{}] made of [{}] nodes", nodeInfos.getClusterName().value(), size()); + } catch (NodeValidationException e) { + try { + IOUtils.close(client, node); + } catch (IOException e1) { + e.addSuppressed(e1); + } + throw new ElasticsearchException(e); } catch (Exception e) { - client.close(); + try { + IOUtils.close(client, node); + } catch (IOException e1) { + e.addSuppressed(e1); + } throw e; } } @@ -150,7 +179,7 @@ public InetSocketAddress[] httpAddresses() { @Override public void close() throws IOException { - client.close(); + IOUtils.close(client, node); } @Override @@ -181,12 +210,12 @@ public void ensureEstimatedStats() { @Override public Iterable getClients() { - return Collections.singleton(client); + return List.of(client); } @Override public NamedWriteableRegistry getNamedWriteableRegistry() { - return client.getNamedWriteableRegistry(); + return node.getNamedWriteableRegistry(); } @Override diff --git a/x-pack/docs/en/security/ccs-clients-integrations/java.asciidoc b/x-pack/docs/en/security/ccs-clients-integrations/java.asciidoc deleted file mode 100644 index a19532bdb67c5..0000000000000 --- a/x-pack/docs/en/security/ccs-clients-integrations/java.asciidoc +++ /dev/null @@ -1,203 +0,0 @@ -[[java-clients]] -=== Java Client and security - -deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.] - -The {es} {security-features} support the Java http://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html[transport client] for Elasticsearch. -The transport client uses the same transport protocol that the cluster nodes use -for inter-node communication. It is very efficient as it does not have to marshall -and unmarshall JSON requests like a typical REST client. - -NOTE: Using the Java Node Client with secured clusters is not recommended or - supported. - -[float] -[[transport-client]] -==== Configuring the Transport Client to work with a Secured Cluster - -To use the transport client with a secured cluster, you need to: - -[[java-transport-client-role]] -. {ref}/setup-xpack-client.html[Configure the {xpack} transport client]. - -. Configure a user with the privileges required to start the transport client. -A default `transport_client` role is built-in to the {es} {security-features}, -which grants the -appropriate cluster permissions for the transport client to work with the secured -cluster. The transport client uses the _Nodes Info API_ to fetch information about -the nodes in the cluster. - -. Set up the transport client. At a minimum, you must configure `xpack.security.user` to -include the name and password of your transport client user in your requests. The -following snippet configures the user credentials globally--every request -submitted with this client includes the `transport_client_user` credentials in -its headers. -+ --- -[source,java] -------------------------------------------------------------------------------------------------- -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -... - -TransportClient client = new PreBuiltXPackTransportClient(Settings.builder() - .put("cluster.name", "myClusterName") - .put("xpack.security.user", "transport_client_user:x-pack-test-password") - ... - .build()) - .addTransportAddress(new TransportAddress("localhost", 9300)) - .addTransportAddress(new TransportAddress("localhost", 9301)); -------------------------------------------------------------------------------------------------- - -WARNING: If you configure a transport client without SSL, passwords are sent in - clear text. - -You can also add an `Authorization` header to each request. If you've configured -global authorization credentials, the `Authorization` header overrides the global -authentication credentials. This is useful when an application has multiple users -who access Elasticsearch using the same client. You can set the global token to -a user that only has the `transport_client` role, and add the `transport_client` -role to the individual users. - -For example, the following snippet adds the `Authorization` header to a search -request: - -[source,java] --------------------------------------------------------------------------------------------------- -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; - -import static UsernamePasswordToken.basicAuthHeaderValue; -... - -TransportClient client = new PreBuiltXPackTransportClient(Settings.builder() - .put("cluster.name", "myClusterName") - .put("xpack.security.user", "transport_client_user:x-pack-test-password") - ... - .build()) - .build() - .addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"), 9300)) - .addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"), 9301)) - -String token = basicAuthHeaderValue("test_user", new SecureString("x-pack-test-password".toCharArray())); - -client.filterWithHeader(Collections.singletonMap("Authorization", token)) - .prepareSearch().get(); --------------------------------------------------------------------------------------------------- --- - -. Enable SSL to authenticate clients and encrypt communications. To enable SSL, -you need to: - -.. Configure the paths to the client's key and certificate in addition to the certificate authorities. -Client authentication requires every client to have a certification signed by a trusted CA. -+ --- -NOTE: Client authentication is enabled by default. For information about - disabling client authentication, see <>. - -[source,java] --------------------------------------------------------------------------------------------------- -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -... - -TransportClient client = new PreBuiltXPackTransportClient(Settings.builder() - .put("cluster.name", "myClusterName") - .put("xpack.security.user", "transport_client_user:x-pack-test-password") - .put("xpack.security.transport.ssl.enabled", true) - .put("xpack.security.transport.ssl.key", "/path/to/client.key") - .put("xpack.security.transport.ssl.certificate", "/path/to/client.crt") - .put("xpack.security.transport.ssl.certificate_authorities", "/path/to/ca.crt") - ... - .build()); --------------------------------------------------------------------------------------------------- --- - -.. Enable the SSL transport by setting `xpack.security.transport.ssl.enabled` to `true` in the -client configuration. -+ --- -[source,java] --------------------------------------------------------------------------------------------------- -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -... - -TransportClient client = new PreBuiltXPackTransportClient(Settings.builder() - .put("cluster.name", "myClusterName") - .put("xpack.security.user", "transport_client_user:x-pack-test-password") - .put("xpack.security.transport.ssl.enabled", true) - .put("xpack.security.transport.ssl.key", "/path/to/client.key") - .put("xpack.security.transport.ssl.certificate", "/path/to/client.crt") - .put("xpack.security.transport.ssl.certificate_authorities", "/path/to/ca.crt") - .put("xpack.security.transport.ssl.enabled", "true") - ... - .build()) - .addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"), 9300)) - .addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"), 9301)) --------------------------------------------------------------------------------------------------- --- - -[float] -[[disabling-client-auth]] -===== Disabling client authentication - -If you want to disable client authentication, you can use a client-specific -transport protocol. For more information see <>. - -If you are not using client authentication and sign the Elasticsearch node -certificates with your own CA, you need to provide the path to the CA -certificate in your client configuration. - -[source,java] ------------------------------------------------------------------------------------------------------- -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -... - -TransportClient client = new PreBuiltXPackTransportClient(Settings.builder() - .put("cluster.name", "myClusterName") - .put("xpack.security.user", "test_user:x-pack-test-password") - .put("xpack.security.transport.ssl.certificate_authorities", "/path/to/ca.crt") - .put("xpack.security.transport.ssl.enabled", "true") - ... - .build()) - .addTransportAddress(new TransportAddress("localhost", 9300)) - .addTransportAddress(new TransportAddress("localhost", 9301)); ------------------------------------------------------------------------------------------------------- - -NOTE: If you are using a public CA that is already trusted by the Java runtime, - you do not need to set the `xpack.security.transport.ssl.certificate_authorities`. - -[float] -[[connecting-anonymously]] -===== Connecting anonymously - -To enable the transport client to connect anonymously, you must assign the -anonymous user the privileges defined in the <> -role. Anonymous access must also be enabled, of course. For more information, -see <>. - -[float] -[[security-client]] -==== Security client - -The {stack} {security-features} expose an API through the `SecurityClient` class. -To get a hold of a `SecurityClient` you first need to create the `XPackClient`, -which is a wrapper around the existing {es} clients (any client class implementing -`org.elasticsearch.client.Client`). - -The following example shows how you can clear the realm caches using -the `SecurityClient`: - -[source,java] ------------------------------------------------------------------------------------------------------- -Client client = ... // create the transport client - -XPackClient xpackClient = new XPackClient(client); -SecurityClient securityClient = xpackClient.security(); -ClearRealmCacheResponse response = securityClient.authc().prepareClearRealmCache() - .realms("ldap1", "ad1") <1> - .usernames("rdeniro") - .get(); ------------------------------------------------------------------------------------------------------- -<1> Clears the `ldap1` and `ad1` realm caches for the `rdeniro` user. diff --git a/x-pack/docs/en/watcher/java.asciidoc b/x-pack/docs/en/watcher/java.asciidoc deleted file mode 100644 index 7224196834f9b..0000000000000 --- a/x-pack/docs/en/watcher/java.asciidoc +++ /dev/null @@ -1,130 +0,0 @@ -[[api-java]] -== Java API - -deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.] - -{xpack} provides a Java client called `WatcherClient` that adds native Java -support for the {watcher}. - -To obtain a `WatcherClient` instance, make sure you first set up the -`XPackClient`. - -[float] -=== Installing XPackClient - -You first need to make sure the +x-pack-transport-{version}+ JAR file is in the classpath. -You can extract this jar from the downloaded {xpack} bundle. - -If you use Maven to manage dependencies, add the following to the `pom.xml`: - -["source","xml",subs="attributes,callouts"] --------------------------------------------------- - - - - - - elasticsearch-releases - https://artifacts.elastic.co/maven - - true - - - false - - - ... - - ... - - - - - org.elasticsearch.client - x-pack-transport - {version} - - ... - - ... - - --------------------------------------------------- - -If you use Gradle, add the dependencies to `build.gradle`: - -["source","groovy",subs="attributes,callouts"] --------------------------------------------------------------- -repositories { - /* ... Any other repositories ... */ - - // Add the Elasticsearch Maven Repository - maven { - name "elastic" - url "https://artifacts.elastic.co/maven" - } -} - -dependencies { - // Provide the x-pack jar on the classpath for compilation and at runtime - compile "org.elasticsearch.client:x-pack-transport:{version}" - - /* ... */ -} --------------------------------------------------------------- - -You can also download the https://artifacts.elastic.co/maven/org/elasticsearch/client/x-pack-transport/{version}/x-pack-transport-{version}.jar[X-Pack Transport JAR] -manually, directly from our Maven repository. - -[float] -=== Obtaining the `WatcherClient` - -To obtain an instance of the `WatcherClient` you first need to create the -`XPackClient`. The `XPackClient` is a wrapper around the standard Java -Elasticsearch `Client`: - -[source,java] --------------------------------------------------- -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -import org.elasticsearch.xpack.core.XPackClient; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.core.watcher.client.WatcherClient; -... - -TransportClient client = new PreBuiltXPackTransportClient(Settings.builder() - .put("cluster.name", "myClusterName") - ... - .build()) - .addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"), 9300)); - -XPackClient xpackClient = new XPackClient(client); -WatcherClient watcherClient = xpackClient.watcher(); --------------------------------------------------- - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/put-watch.asciidoc -include::java/put-watch.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/get-watch.asciidoc -include::java/get-watch.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/delete-watch.asciidoc -include::java/delete-watch.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/execute-watch.asciidoc -include::java/execute-watch.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/ack-watch.asciidoc -include::java/ack-watch.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/activate-watch.asciidoc -include::java/activate-watch.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/deactivate-watch.asciidoc -include::java/deactivate-watch.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/stats.asciidoc -include::java/stats.asciidoc[] - -:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java/service.asciidoc -include::java/service.asciidoc[] diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index 3eda554a84bd4..c74e39f017a3c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -133,7 +133,6 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E private final SetOnce restoreSourceService = new SetOnce<>(); private final SetOnce ccrSettings = new SetOnce<>(); private Client client; - private final boolean transportClientMode; /** * Construct an instance of the CCR container with the specified settings. @@ -155,7 +154,6 @@ public Ccr(final Settings settings) { this.settings = settings; this.enabled = CCR_ENABLED_SETTING.get(settings); this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); - this.transportClientMode = XPackPlugin.transportClientMode(settings); } @Override @@ -340,10 +338,6 @@ public void onIndexModule(IndexModule indexModule) { @Override public Collection createGuiceModules() { - if (transportClientMode) { - return Collections.emptyList(); - } - return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a145569898ee6..6b457ae2fda9e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.core; -import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.ClusterState; @@ -13,12 +12,9 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.license.DeleteLicenseAction; import org.elasticsearch.license.GetBasicStatusAction; import org.elasticsearch.license.GetLicenseAction; @@ -34,8 +30,6 @@ import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.Transport; import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction; import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; @@ -147,8 +141,6 @@ import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus; import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; -import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.security.SecuritySettings; import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction; import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction; @@ -178,9 +170,7 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression; import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges; -import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport; import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage; -import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction; import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction; @@ -197,12 +187,10 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.function.Supplier; +// TODO: merge this into XPackPlugin public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPlugin { static Optional X_PACK_FEATURE = Optional.of("x-pack"); @@ -235,22 +223,6 @@ public List> getSettings() { return settings; } - @Override - public Settings additionalSettings() { - return additionalSettings(settings, XPackSettings.SECURITY_ENABLED.get(settings), XPackPlugin.transportClientMode(settings)); - } - - static Settings additionalSettings(final Settings settings, final boolean enabled, final boolean transportClientMode) { - if (enabled && transportClientMode) { - return Settings.builder() - .put(SecuritySettings.addTransportSettings(settings)) - .put(SecuritySettings.addUserSettings(settings)) - .build(); - } else { - return Settings.EMPTY; - } - } - @Override public List> getClientActions() { return Arrays.asList( @@ -505,27 +477,4 @@ public List getNamedXContent() { DataFrameTransformState::fromXContent) ); } - - @Override - public Map> getTransports( - final Settings settings, - final ThreadPool threadPool, - final PageCacheRecycler pageCacheRecycler, - final CircuitBreakerService circuitBreakerService, - final NamedWriteableRegistry namedWriteableRegistry, - final NetworkService networkService) { - // this should only be used in the transport layer, so do not add it if it is not in transport mode or we are disabled - if (XPackPlugin.transportClientMode(settings) == false || XPackSettings.SECURITY_ENABLED.get(settings) == false) { - return Collections.emptyMap(); - } - final SSLService sslService; - try { - sslService = new SSLService(settings, null); - } catch (Exception e) { - throw new RuntimeException(e); - } - return Collections.singletonMap(SecurityField.NAME4, () -> new SecurityNetty4Transport(settings, Version.CURRENT, threadPool, - networkService, pageCacheRecycler, namedWriteableRegistry, circuitBreakerService, sslService)); - } - } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 2038b35b4e6e0..ababc3c21289a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.inject.Binder; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.multibindings.Multibinder; -import org.elasticsearch.common.inject.util.Providers; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.ClusterSettings; @@ -125,7 +124,6 @@ public Void run() { protected final Settings settings; //private final Environment env; - protected boolean transportClientMode; protected final Licensing licensing; // These should not be directly accessed as they cannot be overridden in tests. Please use the getters so they can be overridden. private static final SetOnce licenseState = new SetOnce<>(); @@ -137,8 +135,7 @@ public XPackPlugin( final Path configPath) { super(settings); this.settings = settings; - this.transportClientMode = transportClientMode(settings); - Environment env = transportClientMode ? null : new Environment(settings, configPath); + Environment env = new Environment(settings, configPath); setSslService(new SSLService(settings, env)); setLicenseState(new XPackLicenseState(settings)); @@ -222,12 +219,7 @@ public Settings additionalSettings() { if (settings.get(xpackInstalledNodeAttrSetting) != null) { throw new IllegalArgumentException("Directly setting [" + xpackInstalledNodeAttrSetting + "] is not permitted"); } - - if (transportClientMode) { - return super.additionalSettings(); - } else { - return Settings.builder().put(super.additionalSettings()).put(xpackInstalledNodeAttrSetting, "true").build(); - } + return Settings.builder().put(super.additionalSettings()).put(xpackInstalledNodeAttrSetting, "true").build(); } @Override @@ -236,10 +228,6 @@ public Collection createGuiceModules() { //modules.add(b -> b.bind(Clock.class).toInstance(getClock())); // used to get core up and running, we do not bind the actual feature set here modules.add(b -> XPackPlugin.createFeatureSetMultiBinder(b, EmptyXPackFeatureSet.class)); - - if (transportClientMode) { - modules.add(b -> b.bind(XPackLicenseState.class).toProvider(Providers.of(null))); - } return modules; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java index 9696ca6e7fde7..2d4991d514027 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.concurrent.CountDownLatch; +@ESIntegTestCase.ClusterScope(transportClientRatio = 0.0) public abstract class AbstractLicensesIntegrationTestCase extends ESIntegTestCase { @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java index 1b7d889d7262a..1f09f959883f3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java @@ -25,7 +25,7 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; -@ESIntegTestCase.ClusterScope(scope = SUITE) +@ESIntegTestCase.ClusterScope(scope = SUITE, transportClientRatio = 0.0) public class StartBasicLicenseTests extends AbstractLicensesIntegrationTestCase { @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java index ca1c361a5b99f..eac145dd0ffa8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java @@ -24,7 +24,7 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; -@ESIntegTestCase.ClusterScope(scope = SUITE) +@ESIntegTestCase.ClusterScope(scope = SUITE, transportClientRatio = 0.0) public class StartTrialLicenseTests extends AbstractLicensesIntegrationTestCase { @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java index 81be978d33103..b03f51d1d195b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java @@ -54,7 +54,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -@ESIntegTestCase.ClusterScope(numDataNodes = 0) +@ESIntegTestCase.ClusterScope(numDataNodes = 0, transportClientRatio = 0.0) public class SourceOnlySnapshotIT extends ESIntegTestCase { @Override diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java index 3a6ab2e5b71d2..122cd570ab108 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java @@ -7,16 +7,36 @@ package org.elasticsearch.xpack.dataframe.integration; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.AcknowledgedResponse; +import org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsRequest; +import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsResponse; +import org.elasticsearch.client.dataframe.PutDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.StartDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.StartDataFrameTransformResponse; +import org.elasticsearch.client.dataframe.StopDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.StopDataFrameTransformResponse; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; +import org.elasticsearch.client.dataframe.transforms.DestConfig; +import org.elasticsearch.client.dataframe.transforms.QueryConfig; +import org.elasticsearch.client.dataframe.transforms.SourceConfig; +import org.elasticsearch.client.dataframe.transforms.pivot.AggregationConfig; +import org.elasticsearch.client.dataframe.transforms.pivot.DateHistogramGroupSource; +import org.elasticsearch.client.dataframe.transforms.pivot.GroupConfig; +import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig; +import org.elasticsearch.client.dataframe.transforms.pivot.SingleGroupSource; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; @@ -26,36 +46,15 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.SecuritySettingsSourceField; -import org.elasticsearch.transport.Netty4Plugin; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.core.XPackClientPlugin; -import org.elasticsearch.xpack.core.dataframe.action.DeleteDataFrameTransformAction; -import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction; -import org.elasticsearch.xpack.core.dataframe.action.PutDataFrameTransformAction; -import org.elasticsearch.xpack.core.dataframe.action.StartDataFrameTransformAction; -import org.elasticsearch.xpack.core.dataframe.action.StopDataFrameTransformAction; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.QueryConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.SourceConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.pivot.AggregationConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.pivot.DateHistogramGroupSource; -import org.elasticsearch.xpack.core.dataframe.transforms.pivot.GroupConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.pivot.SingleGroupSource; -import org.elasticsearch.xpack.core.security.SecurityField; - -import java.net.URISyntaxException; -import java.nio.file.Path; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.ZoneId; -import java.util.Arrays; -import java.util.Collection; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -64,18 +63,18 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.core.Is.is; -abstract class DataFrameIntegTestCase extends ESIntegTestCase { +abstract class DataFrameIntegTestCase extends ESRestTestCase { protected static final String REVIEWS_INDEX_NAME = "data_frame_reviews"; private Map transformConfigs = new HashMap<>(); - protected void cleanUp() { + protected void cleanUp() throws IOException { cleanUpTransforms(); waitForPendingTasks(); } - protected void cleanUpTransforms() { + protected void cleanUpTransforms() throws IOException { for (DataFrameTransformConfig config : transformConfigs.values()) { stopDataFrameTransform(config.getId()); deleteDataFrameTransform(config.getId()); @@ -83,41 +82,42 @@ protected void cleanUpTransforms() { transformConfigs.clear(); } - protected StopDataFrameTransformAction.Response stopDataFrameTransform(String id) { - return client().execute(StopDataFrameTransformAction.INSTANCE, - new StopDataFrameTransformAction.Request(id, true, false, null)).actionGet(); + protected StopDataFrameTransformResponse stopDataFrameTransform(String id) throws IOException { + RestHighLevelClient restClient = new TestRestHighLevelClient(); + return restClient.dataFrame().stopDataFrameTransform(new StopDataFrameTransformRequest(id, true, null), RequestOptions.DEFAULT); } - protected StartDataFrameTransformAction.Response startDataFrameTransform(String id) { - return client().execute(StartDataFrameTransformAction.INSTANCE, - new StartDataFrameTransformAction.Request(id, false)).actionGet(); + protected StartDataFrameTransformResponse startDataFrameTransform(String id, RequestOptions options) throws IOException { + RestHighLevelClient restClient = new TestRestHighLevelClient(); + return restClient.dataFrame().startDataFrameTransform(new StartDataFrameTransformRequest(id), options); } - protected AcknowledgedResponse deleteDataFrameTransform(String id) { - AcknowledgedResponse response = client().execute(DeleteDataFrameTransformAction.INSTANCE, - new DeleteDataFrameTransformAction.Request(id)) - .actionGet(); + protected AcknowledgedResponse deleteDataFrameTransform(String id) throws IOException { + RestHighLevelClient restClient = new TestRestHighLevelClient(); + AcknowledgedResponse response = + restClient.dataFrame().deleteDataFrameTransform(new DeleteDataFrameTransformRequest(id), RequestOptions.DEFAULT); if (response.isAcknowledged()) { transformConfigs.remove(id); } return response; } - protected AcknowledgedResponse putDataFrameTransform(DataFrameTransformConfig config) { + protected AcknowledgedResponse putDataFrameTransform(DataFrameTransformConfig config, RequestOptions options) throws IOException { if (transformConfigs.keySet().contains(config.getId())) { throw new IllegalArgumentException("data frame transform [" + config.getId() + "] is already registered"); } - AcknowledgedResponse response = client().execute(PutDataFrameTransformAction.INSTANCE, - new PutDataFrameTransformAction.Request(config)) - .actionGet(); + RestHighLevelClient restClient = new TestRestHighLevelClient(); + AcknowledgedResponse response = + restClient.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(config), options); if (response.isAcknowledged()) { transformConfigs.put(config.getId(), config); } return response; } - protected GetDataFrameTransformsStatsAction.Response getDataFrameTransformStats(String id) { - return client().execute(GetDataFrameTransformsStatsAction.INSTANCE, new GetDataFrameTransformsStatsAction.Request(id)).actionGet(); + protected GetDataFrameTransformStatsResponse getDataFrameTransformStats(String id) throws IOException { + RestHighLevelClient restClient = new TestRestHighLevelClient(); + return restClient.dataFrame().getDataFrameTransformStats(new GetDataFrameTransformStatsRequest(id), RequestOptions.DEFAULT); } protected void waitUntilCheckpoint(String id, long checkpoint) throws Exception { @@ -136,38 +136,40 @@ protected void waitUntilCheckpoint(String id, long checkpoint, TimeValue waitTim } protected DateHistogramGroupSource createDateHistogramGroupSource(String field, long interval, ZoneId zone, String format) { - DateHistogramGroupSource source = new DateHistogramGroupSource(field); - source.setFormat(format); - source.setInterval(interval); - source.setTimeZone(zone); - return source; + DateHistogramGroupSource.Builder builder = DateHistogramGroupSource.builder() + .setField(field) + .setFormat(format) + .setInterval(interval) + .setTimeZone(zone); + return builder.build(); } protected DateHistogramGroupSource createDateHistogramGroupSource(String field, DateHistogramInterval interval, ZoneId zone, String format) { - DateHistogramGroupSource source = new DateHistogramGroupSource(field); - source.setFormat(format); - source.setDateHistogramInterval(interval); - source.setTimeZone(zone); - return source; + DateHistogramGroupSource.Builder builder = DateHistogramGroupSource.builder() + .setField(field) + .setFormat(format) + .setDateHistgramInterval(interval) + .setTimeZone(zone); + return builder.build(); } protected GroupConfig createGroupConfig(Map groups) throws Exception { - Map lazyParsed = new HashMap<>(groups.size()); - for(Map.Entry sgs : groups.entrySet()) { - lazyParsed.put(sgs.getKey(), Collections.singletonMap(sgs.getValue().getType().value(), toLazy(sgs.getValue()))); + GroupConfig.Builder builder = GroupConfig.builder(); + for (Map.Entry sgs : groups.entrySet()) { + builder.groupBy(sgs.getKey(), sgs.getValue()); } - return new GroupConfig(lazyParsed, groups); + return builder.build(); } protected QueryConfig createQueryConfig(QueryBuilder queryBuilder) throws Exception { - return new QueryConfig(toLazy(queryBuilder), queryBuilder); + return new QueryConfig(queryBuilder); } protected AggregationConfig createAggConfig(AggregatorFactories.Builder aggregations) throws Exception { - return new AggregationConfig(toLazy(aggregations), aggregations); + return new AggregationConfig(aggregations); } protected PivotConfig createPivotConfig(Map groups, @@ -178,7 +180,11 @@ protected PivotConfig createPivotConfig(Map groups, protected PivotConfig createPivotConfig(Map groups, AggregatorFactories.Builder aggregations, Integer size) throws Exception { - return new PivotConfig(createGroupConfig(groups), createAggConfig(aggregations), size); + PivotConfig.Builder builder = PivotConfig.builder() + .setGroups(createGroupConfig(groups)) + .setAggregationConfig(createAggConfig(aggregations)) + .setMaxPageSearchSize(size); + return builder.build(); } protected DataFrameTransformConfig createTransformConfig(String id, @@ -195,16 +201,18 @@ protected DataFrameTransformConfig createTransformConfig(String id, String destinationIndex, QueryBuilder queryBuilder, String... sourceIndices) throws Exception { - return new DataFrameTransformConfig(id, - new SourceConfig(sourceIndices, createQueryConfig(queryBuilder)), - new DestConfig(destinationIndex), - Collections.emptyMap(), - createPivotConfig(groups, aggregations), - "Test data frame transform config id: " + id); + return DataFrameTransformConfig.builder() + .setId(id) + .setSource(SourceConfig.builder().setIndex(sourceIndices).setQueryConfig(createQueryConfig(queryBuilder)).build()) + .setDest(new DestConfig(destinationIndex)) + .setPivotConfig(createPivotConfig(groups, aggregations)) + .setDescription("Test data frame transform config id: " + id) + .build(); } protected void createReviewsIndex() throws Exception { final int numDocs = 1000; + RestHighLevelClient restClient = new TestRestHighLevelClient(); // create mapping try (XContentBuilder builder = jsonBuilder()) { @@ -229,16 +237,13 @@ protected void createReviewsIndex() throws Exception { .endObject(); } builder.endObject(); - CreateIndexResponse response = client().admin() - .indices() - .prepareCreate(REVIEWS_INDEX_NAME) - .addMapping("_doc", builder) - .get(); + CreateIndexResponse response = + restClient.indices().create(new CreateIndexRequest(REVIEWS_INDEX_NAME).mapping(builder), RequestOptions.DEFAULT); assertThat(response.isAcknowledged(), is(true)); } // create index - BulkRequestBuilder bulk = client().prepareBulk(REVIEWS_INDEX_NAME, "_doc"); + BulkRequest bulk = new BulkRequest(REVIEWS_INDEX_NAME); int day = 10; for (int i = 0; i < numDocs; i++) { long user = i % 28; @@ -267,15 +272,15 @@ protected void createReviewsIndex() throws Exception { bulk.add(new IndexRequest().source(sourceBuilder.toString(), XContentType.JSON)); if (i % 50 == 0) { - BulkResponse response = client().bulk(bulk.request()).get(); + BulkResponse response = restClient.bulk(bulk, RequestOptions.DEFAULT); assertThat(response.buildFailureMessage(), response.hasFailures(), is(false)); - bulk = client().prepareBulk(REVIEWS_INDEX_NAME, "_doc"); + bulk = new BulkRequest(REVIEWS_INDEX_NAME); day += 1; } } - BulkResponse response = client().bulk(bulk.request()).get(); + BulkResponse response = restClient.bulk(bulk, RequestOptions.DEFAULT); assertThat(response.buildFailureMessage(), response.hasFailures(), is(false)); - client().admin().indices().prepareRefresh(REVIEWS_INDEX_NAME).get(); + restClient.indices().refresh(new RefreshRequest(REVIEWS_INDEX_NAME), RequestOptions.DEFAULT); } protected Map toLazy(ToXContent parsedObject) throws Exception { @@ -293,8 +298,9 @@ private void waitForPendingTasks() { listTasksRequest.setWaitForCompletion(true); listTasksRequest.setDetailed(true); listTasksRequest.setTimeout(TimeValue.timeValueSeconds(10)); + RestHighLevelClient restClient = new TestRestHighLevelClient(); try { - admin().cluster().listTasks(listTasksRequest).get(); + restClient.tasks().list(listTasksRequest, RequestOptions.DEFAULT); } catch (Exception e) { throw new AssertionError("Failed to wait for pending tasks to complete", e); } @@ -307,33 +313,17 @@ protected NamedXContentRegistry xContentRegistry() { } @Override - protected Settings externalClusterClientSettings() { - Path key; - Path certificate; - try { - key = PathUtils.get(getClass().getResource("/testnode.pem").toURI()); - certificate = PathUtils.get(getClass().getResource("/testnode.crt").toURI()); - } catch (URISyntaxException e) { - throw new IllegalStateException("error trying to get keystore path", e); - } - Settings.Builder builder = Settings.builder(); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4); - builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING); - builder.put("xpack.security.transport.ssl.enabled", true); - builder.put("xpack.security.transport.ssl.key", key.toAbsolutePath().toString()); - builder.put("xpack.security.transport.ssl.certificate", certificate.toAbsolutePath().toString()); - builder.put("xpack.security.transport.ssl.key_passphrase", "testnode"); - builder.put("xpack.security.transport.ssl.verification_mode", "certificate"); - return builder.build(); - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, Netty4Plugin.class); + protected Settings restClientSettings() { + final String token = "Basic " + + Base64.getEncoder().encodeToString(("x_pack_rest_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8)); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); } - @Override - protected Collection> transportClientPlugins() { - return Arrays.asList(XPackClientPlugin.class, Netty4Plugin.class); + private class TestRestHighLevelClient extends RestHighLevelClient { + TestRestHighLevelClient() { + super(client(), restClient -> {}, Collections.emptyList()); + } } } diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java index cc2e8c4436e06..363218d1b0f14 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java @@ -6,16 +6,18 @@ package org.elasticsearch.xpack.dataframe.integration; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.core.IndexerState; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.client.dataframe.transforms.pivot.SingleGroupSource; +import org.elasticsearch.client.dataframe.transforms.pivot.TermsGroupSource; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; -import org.elasticsearch.xpack.core.dataframe.transforms.pivot.SingleGroupSource; -import org.elasticsearch.xpack.core.dataframe.transforms.pivot.TermsGroupSource; -import org.elasticsearch.xpack.core.indexing.IndexerState; import org.junit.After; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -24,7 +26,7 @@ public class DataFrameTransformIT extends DataFrameIntegTestCase { @After - public void cleanTransforms() { + public void cleanTransforms() throws IOException { cleanUp(); } @@ -34,8 +36,8 @@ public void testDataFrameTransformCrud() throws Exception { Map groups = new HashMap<>(); groups.put("by-day", createDateHistogramGroupSource("timestamp", DateHistogramInterval.DAY, null, null)); - groups.put("by-user", new TermsGroupSource("user_id")); - groups.put("by-business", new TermsGroupSource("business_id")); + groups.put("by-user", TermsGroupSource.builder().setField("user_id").build()); + groups.put("by-business", TermsGroupSource.builder().setField("business_id").build()); AggregatorFactories.Builder aggs = AggregatorFactories.builder() .addAggregator(AggregationBuilders.avg("review_score").field("stars")) @@ -47,8 +49,10 @@ public void testDataFrameTransformCrud() throws Exception { "reviews-by-user-business-day", REVIEWS_INDEX_NAME); - assertTrue(putDataFrameTransform(config).isAcknowledged()); - assertTrue(startDataFrameTransform(config.getId()).isStarted()); + final RequestOptions options = + expectWarnings("[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future."); + assertTrue(putDataFrameTransform(config, options).isAcknowledged()); + assertTrue(startDataFrameTransform(config.getId(), options).isStarted()); waitUntilCheckpoint(config.getId(), 1L); diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java index 7d0fb179a2228..d6ef3cc641be2 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java @@ -7,24 +7,23 @@ package org.elasticsearch.xpack.dataframe.integration; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.SecuritySettingsSourceField; -import org.elasticsearch.transport.Netty4Plugin; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig; @@ -34,11 +33,10 @@ import org.elasticsearch.xpack.core.dataframe.transforms.pivot.GroupConfig; import org.elasticsearch.xpack.core.dataframe.transforms.pivot.HistogramGroupSource; import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig; -import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.dataframe.transforms.TransformProgressGatherer; -import java.util.Arrays; -import java.util.Collection; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Collections; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -47,10 +45,11 @@ import static org.hamcrest.Matchers.is; @LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344") -public class DataFrameTransformProgressIT extends ESIntegTestCase { +public class DataFrameTransformProgressIT extends ESRestTestCase { protected void createReviewsIndex() throws Exception { final int numDocs = 1000; + final RestHighLevelClient restClient = new TestRestHighLevelClient(); // create mapping try (XContentBuilder builder = jsonBuilder()) { @@ -75,16 +74,13 @@ protected void createReviewsIndex() throws Exception { .endObject(); } builder.endObject(); - CreateIndexResponse response = client().admin() - .indices() - .prepareCreate(REVIEWS_INDEX_NAME) - .addMapping("_doc", builder) - .get(); + CreateIndexResponse response = restClient.indices() + .create(new CreateIndexRequest(REVIEWS_INDEX_NAME).mapping(builder), RequestOptions.DEFAULT); assertThat(response.isAcknowledged(), is(true)); } // create index - BulkRequestBuilder bulk = client().prepareBulk(REVIEWS_INDEX_NAME, "_doc"); + BulkRequest bulk = new BulkRequest(REVIEWS_INDEX_NAME); int day = 10; for (int i = 0; i < numDocs; i++) { long user = i % 28; @@ -113,14 +109,14 @@ protected void createReviewsIndex() throws Exception { bulk.add(new IndexRequest().source(sourceBuilder.toString(), XContentType.JSON)); if (i % 50 == 0) { - BulkResponse response = client().bulk(bulk.request()).get(); + BulkResponse response = restClient.bulk(bulk, RequestOptions.DEFAULT); assertThat(response.buildFailureMessage(), response.hasFailures(), is(false)); - bulk = client().prepareBulk(REVIEWS_INDEX_NAME, "_doc"); + bulk = new BulkRequest(REVIEWS_INDEX_NAME); day += 1; } } - client().bulk(bulk.request()).get(); - client().admin().indices().prepareRefresh(REVIEWS_INDEX_NAME).get(); + restClient.bulk(bulk, RequestOptions.DEFAULT); + restClient.indices().refresh(new RefreshRequest(REVIEWS_INDEX_NAME), RequestOptions.DEFAULT); } public void testGetProgress() throws Exception { @@ -140,10 +136,11 @@ public void testGetProgress() throws Exception { pivotConfig, null); - PlainActionFuture progressFuture = new PlainActionFuture<>(); - TransformProgressGatherer.getInitialProgress(client(), config, progressFuture); + final RestHighLevelClient restClient = new TestRestHighLevelClient(); + SearchResponse response = restClient.search(TransformProgressGatherer.getSearchRequest(config), RequestOptions.DEFAULT); - DataFrameTransformProgress progress = progressFuture.get(); + DataFrameTransformProgress progress = + TransformProgressGatherer.searchResponseToDataFrameTransformProgressFunction().apply(response); assertThat(progress.getTotalDocs(), equalTo(1000L)); assertThat(progress.getRemainingDocs(), equalTo(1000L)); @@ -160,34 +157,28 @@ public void testGetProgress() throws Exception { pivotConfig, null); - - progressFuture = new PlainActionFuture<>(); - - TransformProgressGatherer.getInitialProgress(client(), config, progressFuture); - progress = progressFuture.get(); + response = restClient.search(TransformProgressGatherer.getSearchRequest(config), RequestOptions.DEFAULT); + progress = TransformProgressGatherer.searchResponseToDataFrameTransformProgressFunction().apply(response); assertThat(progress.getTotalDocs(), equalTo(35L)); assertThat(progress.getRemainingDocs(), equalTo(35L)); assertThat(progress.getPercentComplete(), equalTo(0.0)); - client().admin().indices().prepareDelete(REVIEWS_INDEX_NAME).get(); + deleteIndex(REVIEWS_INDEX_NAME); } @Override - protected Settings externalClusterClientSettings() { - Settings.Builder builder = Settings.builder(); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4); - builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING); - return builder.build(); + protected Settings restClientSettings() { + final String token = "Basic " + + Base64.getEncoder().encodeToString(("x_pack_rest_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8)); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); } - @Override - protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, Netty4Plugin.class); - } - - @Override - protected Collection> transportClientPlugins() { - return Arrays.asList(XPackClientPlugin.class, Netty4Plugin.class); + private class TestRestHighLevelClient extends RestHighLevelClient { + TestRestHighLevelClient() { + super(client(), restClient -> {}, Collections.emptyList()); + } } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java index 34343e5fe8820..e8206311c012b 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java @@ -92,7 +92,6 @@ public class DataFrame extends Plugin implements ActionPlugin, PersistentTaskPlu private final boolean enabled; private final Settings settings; - private final boolean transportClientMode; private final SetOnce dataFrameTransformsConfigManager = new SetOnce<>(); private final SetOnce dataFrameAuditor = new SetOnce<>(); private final SetOnce dataFrameTransformsCheckpointService = new SetOnce<>(); @@ -100,19 +99,12 @@ public class DataFrame extends Plugin implements ActionPlugin, PersistentTaskPlu public DataFrame(Settings settings) { this.settings = settings; - this.enabled = XPackSettings.DATA_FRAME_ENABLED.get(settings); - this.transportClientMode = XPackPlugin.transportClientMode(settings); } @Override public Collection createGuiceModules() { List modules = new ArrayList<>(); - - if (transportClientMode) { - return modules; - } - modules.add(b -> XPackPlugin.bindFeatureSet(b, DataFrameFeatureSet.class)); return modules; } @@ -159,7 +151,7 @@ public List getRestHandlers(final Settings settings, final RestCont @Override public List> getExecutorBuilders(Settings settings) { - if (false == enabled || transportClientMode) { + if (false == enabled) { return emptyList(); } @@ -173,7 +165,7 @@ public List> getExecutorBuilders(Settings settings) { public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - if (enabled == false || transportClientMode) { + if (enabled == false) { return emptyList(); } dataFrameAuditor.set(new DataFrameAuditor(client, clusterService.getNodeName())); @@ -203,7 +195,7 @@ public UnaryOperator> getIndexTemplateMetaDat @Override public List> getPersistentTasksExecutor(ClusterService clusterService, ThreadPool threadPool, Client client, SettingsModule settingsModule) { - if (enabled == false || transportClientMode) { + if (enabled == false) { return emptyList(); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java index 23168627d442e..18a341e217294 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java @@ -11,10 +11,13 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; +import java.util.function.Function; + /** * Utility class to gather the progress information for a given config and its cursor position */ @@ -29,17 +32,10 @@ public final class TransformProgressGatherer { public static void getInitialProgress(Client client, DataFrameTransformConfig config, ActionListener progressListener) { - SearchRequest request = client.prepareSearch(config.getSource().getIndex()) - .setSize(0) - .setAllowPartialSearchResults(false) - .setTrackTotalHits(true) - .setQuery(config.getSource().getQueryConfig().getQuery()) - .request(); + SearchRequest request = getSearchRequest(config); ActionListener searchResponseActionListener = ActionListener.wrap( - searchResponse -> { - progressListener.onResponse(new DataFrameTransformProgress(searchResponse.getHits().getTotalHits().value, null)); - }, + searchResponse -> progressListener.onResponse(searchResponseToDataFrameTransformProgressFunction().apply(searchResponse)), progressListener::onFailure ); ClientHelper.executeWithHeadersAsync(config.getHeaders(), @@ -50,4 +46,17 @@ public static void getInitialProgress(Client client, searchResponseActionListener); } + public static SearchRequest getSearchRequest(DataFrameTransformConfig config) { + SearchRequest request = new SearchRequest(config.getSource().getIndex()); + request.allowPartialSearchResults(false); + request.source(new SearchSourceBuilder() + .size(0) + .trackTotalHits(true) + .query(config.getSource().getQueryConfig().getQuery())); + return request; + } + + public static Function searchResponseToDataFrameTransformProgressFunction() { + return searchResponse -> new DataFrameTransformProgress(searchResponse.getHits().getTotalHits().value, null); + } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java index 2e7d2fbbc555d..0b6c6001ca0ef 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java @@ -93,12 +93,10 @@ public class IndexLifecycle extends Plugin implements ActionPlugin { private final SetOnce indexLifecycleInitialisationService = new SetOnce<>(); private Settings settings; private boolean enabled; - private boolean transportClientMode; public IndexLifecycle(Settings settings) { this.settings = settings; this.enabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings); - this.transportClientMode = XPackPlugin.transportClientMode(settings); } // overridable by tests @@ -108,13 +106,7 @@ protected Clock getClock() { public Collection createGuiceModules() { List modules = new ArrayList<>(); - - if (transportClientMode) { - return modules; - } - modules.add(b -> XPackPlugin.bindFeatureSet(b, IndexLifecycleFeatureSet.class)); - return modules; } @@ -132,7 +124,7 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - if (enabled == false || transportClientMode) { + if (enabled == false) { return emptyList(); } indexLifecycleInitialisationService.set(new IndexLifecycleService(settings, client, clusterService, threadPool, diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java index a1a37beb1d129..673c10f885447 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java @@ -77,7 +77,7 @@ import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsNull.nullValue; -@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0) +@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0.0) public class IndexLifecycleInitialisationTests extends ESIntegTestCase { private Settings settings; private LifecyclePolicy lifecyclePolicy; @@ -109,29 +109,11 @@ protected boolean ignoreExternalCluster() { return true; } - @Override - protected Settings transportClientSettings() { - Settings.Builder settings = Settings.builder().put(super.transportClientSettings()); - settings.put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true); - settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); - settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); - settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false); - settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); - settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false); - return settings.build(); - } - @Override protected Collection> nodePlugins() { return Arrays.asList(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class, TestILMPlugin.class); } - @Override - protected Collection> transportClientPlugins() { - return nodePlugins(); - } - @Before public void init() { settings = Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1) diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index d3ca4dd0098b7..9de1c2f56d1fd 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -36,21 +36,15 @@ public class Logstash extends Plugin implements ActionPlugin { Pattern.quote("${logstash.template.version}"); private final boolean enabled; - private final boolean transportClientMode; public Logstash(Settings settings) { this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings); - this.transportClientMode = XPackPlugin.transportClientMode(settings); } boolean isEnabled() { return enabled; } - boolean isTransportClient() { - return transportClientMode; - } - public Collection createGuiceModules() { List modules = new ArrayList<>(); modules.add(b -> { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 89b2ec81f87ef..128bcce67994b 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -16,7 +16,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.reindex.ReindexPlugin; +import org.elasticsearch.license.LicenseService; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.plugins.Plugin; @@ -24,9 +26,8 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.transport.Netty4Plugin; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; @@ -35,9 +36,12 @@ import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.authc.TokenMetaData; +import org.elasticsearch.xpack.ml.LocalStateMachineLearning; import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URISyntaxException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -45,9 +49,12 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.test.XContentTestUtils.convertToMap; import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile; /** * Base class of ML integration tests that use a native autodetect process @@ -62,16 +69,34 @@ protected NamedXContentRegistry xContentRegistry() { @Override protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, Netty4Plugin.class); + return Arrays.asList(LocalStateMachineLearning.class, Netty4Plugin.class, ReindexPlugin.class); } @Override - protected Collection> transportClientPlugins() { - return Arrays.asList(XPackClientPlugin.class, Netty4Plugin.class, ReindexPlugin.class); + protected Function getClientWrapper() { + final Map headers = + Map.of("Authorization", basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); + // we need to wrap node clients because we do not specify a user for nodes and all requests will use the system + // user. This is ok for internal n2n stuff but the test framework does other things like wiping indices, repositories, etc + // that the system user cannot do. so we wrap the node client with a user that can do these things since the client() calls + // are randomized to return both node clients and transport clients + // transport clients do not need to be wrapped since we specify the xpack.security.user setting that sets the default user to be + // used for the transport client. If we did not set a default user then the transport client would not even be allowed + // to connect + return client -> client.filterWithHeader(headers); } - @Override protected Settings externalClusterClientSettings() { + final Path home = createTempDir(); + final Path xpackConf = home.resolve("config"); + try { + Files.createDirectories(xpackConf); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + writeFile(xpackConf, "users", "x_pack_rest_user" + ":" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING + "\n"); + writeFile(xpackConf, "users_roles", "superuser:x_pack_rest_user\n"); + Path key; Path certificate; try { @@ -80,10 +105,17 @@ protected Settings externalClusterClientSettings() { } catch (URISyntaxException e) { throw new IllegalStateException("error trying to get keystore path", e); } + Settings.Builder builder = Settings.builder(); + builder.put("node.ml", false); builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4); - builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING); builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true); + builder.put(XPackSettings.SECURITY_ENABLED.getKey(), true); + builder.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false); + builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false); + builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false); + builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); + builder.put(Environment.PATH_HOME_SETTING.getKey(), home); builder.put("xpack.security.transport.ssl.enabled", true); builder.put("xpack.security.transport.ssl.key", key.toAbsolutePath().toString()); builder.put("xpack.security.transport.ssl.certificate", certificate.toAbsolutePath().toString()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index f679170bc673d..42e945ffec8fd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -305,7 +305,6 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu private final Settings settings; private final Environment env; private final boolean enabled; - private final boolean transportClientMode; private final SetOnce autodetectProcessManager = new SetOnce<>(); private final SetOnce datafeedManager = new SetOnce<>(); @@ -314,8 +313,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu public MachineLearning(Settings settings, Path configPath) { this.settings = settings; this.enabled = XPackSettings.MACHINE_LEARNING_ENABLED.get(settings); - this.transportClientMode = XPackPlugin.transportClientMode(settings); - this.env = transportClientMode ? null : new Environment(settings, configPath); + this.env = new Environment(settings, configPath); } protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } @@ -349,7 +347,7 @@ public Settings additionalSettings() { String maxOpenJobsPerNodeNodeAttrName = "node.attr." + MAX_OPEN_JOBS_NODE_ATTR; String machineMemoryAttrName = "node.attr." + MACHINE_MEMORY_NODE_ATTR; - if (enabled == false || transportClientMode) { + if (enabled == false) { disallowMlNodeAttributes(mlEnabledNodeAttrName, maxOpenJobsPerNodeNodeAttrName, machineMemoryAttrName); return Settings.EMPTY; } @@ -405,7 +403,7 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - if (enabled == false || transportClientMode) { + if (enabled == false) { // special holder for @link(MachineLearningFeatureSetUsage) which needs access to job manager, empty if ML is disabled return Collections.singletonList(new JobManagerHolder()); } @@ -506,7 +504,7 @@ public List> getPersistentTasksExecutor(ClusterServic ThreadPool threadPool, Client client, SettingsModule settingsModule) { - if (enabled == false || transportClientMode) { + if (enabled == false) { return emptyList(); } @@ -519,15 +517,9 @@ public List> getPersistentTasksExecutor(ClusterServic public Collection createGuiceModules() { List modules = new ArrayList<>(); - - if (transportClientMode) { - return modules; - } - modules.add(b -> { XPackPlugin.bindFeatureSet(b, MachineLearningFeatureSet.class); }); - return modules; } @@ -650,7 +642,7 @@ public List getRestHandlers(Settings settings, RestController restC @Override public List> getExecutorBuilders(Settings settings) { - if (false == enabled || transportClientMode) { + if (false == enabled) { return emptyList(); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java index bcfab50c21e00..2a5b4369cdf9a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java @@ -22,7 +22,6 @@ import org.elasticsearch.plugins.Platforms; import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.XPackField; -import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; @@ -76,7 +75,7 @@ public MachineLearningFeatureSet(Environment environment, ClusterService cluster // Don't try to get the native code version if ML is disabled - it causes too much controversy // if ML has been disabled because of some OS incompatibility. Also don't try to get the native // code version in the transport client - the controller process won't be running. - if (enabled && XPackPlugin.transportClientMode(environment.settings()) == false) { + if (enabled) { try { if (isRunningOnMlPlatform(true)) { NativeController nativeController = NativeControllerHolder.getNativeController(clusterService.getNodeName(), diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java index e2b137bd95d19..0472d909d0fa9 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java @@ -83,11 +83,9 @@ public class Monitoring extends Plugin implements ActionPlugin { protected final Settings settings; private final boolean enabled; - private final boolean transportClientMode; public Monitoring(Settings settings) { this.settings = settings; - this.transportClientMode = XPackPlugin.transportClientMode(settings); this.enabled = XPackSettings.MONITORING_ENABLED.get(settings); } @@ -100,16 +98,12 @@ boolean isEnabled() { return enabled; } - boolean isTransportClient() { - return transportClientMode; - } - @Override public Collection createGuiceModules() { List modules = new ArrayList<>(); modules.add(b -> { XPackPlugin.bindFeatureSet(b, MonitoringFeatureSet.class); - if (transportClientMode || enabled == false) { + if (enabled == false) { b.bind(MonitoringService.class).toProvider(Providers.of(null)); b.bind(Exporters.class).toProvider(Providers.of(null)); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringPluginClientTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringPluginClientTests.java index 6fb967c79782e..b4dc9b3112ece 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringPluginClientTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringPluginClientTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.monitoring; import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -14,17 +13,6 @@ public class MonitoringPluginClientTests extends ESTestCase { - public void testModulesWithClientSettings() throws Exception { - Settings settings = Settings.builder() - .put("path.home", createTempDir()) - .put(Client.CLIENT_TYPE_SETTING_S.getKey(), TransportClient.CLIENT_TYPE) - .build(); - - Monitoring plugin = new Monitoring(settings); - assertThat(plugin.isEnabled(), is(true)); - assertThat(plugin.isTransportClient(), is(true)); - } - public void testModulesWithNodeSettings() throws Exception { // these settings mimic what ES does when running as a node... Settings settings = Settings.builder() @@ -33,6 +21,5 @@ public void testModulesWithNodeSettings() throws Exception { .build(); Monitoring plugin = new Monitoring(settings); assertThat(plugin.isEnabled(), is(true)); - assertThat(plugin.isTransportClient(), is(false)); } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java index 8ebbf1bccf864..faa713efb7d11 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java @@ -108,12 +108,10 @@ public class Rollup extends Plugin implements ActionPlugin, PersistentTaskPlugin private final SetOnce schedulerEngine = new SetOnce<>(); private final Settings settings; private final boolean enabled; - private final boolean transportClientMode; public Rollup(Settings settings) { this.settings = settings; this.enabled = XPackSettings.ROLLUP_ENABLED.get(settings); - this.transportClientMode = XPackPlugin.transportClientMode(settings); } @Override @@ -127,10 +125,6 @@ public Collection createComponents(Client client, ClusterService cluster @Override public Collection createGuiceModules() { List modules = new ArrayList<>(); - - if (transportClientMode) { - return modules; - } modules.add(b -> XPackPlugin.bindFeatureSet(b, RollupFeatureSet.class)); return modules; } @@ -178,7 +172,7 @@ public List getRestHandlers(Settings settings, RestController restC @Override public List> getExecutorBuilders(Settings settings) { - if (false == enabled || transportClientMode) { + if (false == enabled) { return emptyList(); } @@ -193,7 +187,7 @@ public List> getPersistentTasksExecutor(ClusterServic ThreadPool threadPool, Client client, SettingsModule settingsModule) { - if (enabled == false || transportClientMode ) { + if (enabled == false) { return emptyList(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index a6218522fb7e5..c7ada6e79a9ac 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -270,7 +270,6 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw private final Settings settings; private final Environment env; private final boolean enabled; - private final boolean transportClientMode; /* what a PITA that we need an extra indirection to initialize this. Yet, once we got rid of guice we can thing about how * to fix this or make it simpler. Today we need several service that are created in createComponents but we need to register * an instance of TransportInterceptor way earlier before createComponents is called. */ @@ -293,10 +292,9 @@ public Security(Settings settings, final Path configPath) { Security(Settings settings, final Path configPath, List extensions) { this.settings = settings; - this.transportClientMode = XPackPlugin.transportClientMode(settings); - this.env = transportClientMode ? null : new Environment(settings, configPath); + this.env = new Environment(settings, configPath); this.enabled = XPackSettings.SECURITY_ENABLED.get(settings); - if (enabled && transportClientMode == false) { + if (enabled) { runStartupChecks(settings); // we load them all here otherwise we can't access secure settings since they are closed once the checks are // fetched @@ -327,24 +325,11 @@ private static void runStartupChecks(Settings settings) { @Override public Collection createGuiceModules() { List modules = new ArrayList<>(); - if (enabled == false || transportClientMode) { + if (enabled == false) { modules.add(b -> b.bind(IPFilter.class).toProvider(Providers.of(null))); } - - if (transportClientMode) { - if (enabled == false) { - return modules; - } - modules.add(b -> { - // for transport client we still must inject these ssl classes with guice - b.bind(SSLService.class).toInstance(getSslService()); - }); - - return modules; - } modules.add(b -> XPackPlugin.bindFeatureSet(b, SecurityFeatureSet.class)); - if (enabled == false) { modules.add(b -> { b.bind(Realms.class).toProvider(Providers.of(null)); // for SecurityFeatureSet @@ -568,12 +553,12 @@ private AuthenticationFailureHandler createAuthenticationFailureHandler(final Re @Override public Settings additionalSettings() { - return additionalSettings(settings, enabled, transportClientMode); + return additionalSettings(settings, enabled); } // visible for tests - static Settings additionalSettings(final Settings settings, final boolean enabled, final boolean transportClientMode) { - if (enabled && transportClientMode == false) { + static Settings additionalSettings(final Settings settings, final boolean enabled) { + if (enabled) { final Settings.Builder builder = Settings.builder(); builder.put(SecuritySettings.addTransportSettings(settings)); @@ -606,19 +591,15 @@ static Settings additionalSettings(final Settings settings, final boolean enable @Override public List> getSettings() { - return getSettings(transportClientMode, securityExtensions); + return getSettings(securityExtensions); } /** * Get the {@link Setting setting configuration} for all security components, including those defined in extensions. */ - public static List> getSettings(boolean transportClientMode, List securityExtensions) { + public static List> getSettings(List securityExtensions) { List> settingsList = new ArrayList<>(); - if (transportClientMode) { - return settingsList; - } - // The following just apply in node mode settingsList.add(XPackSettings.FIPS_MODE_ENABLED); @@ -657,9 +638,6 @@ public static List> getSettings(boolean transportClientMode, List getRestHeaders() { - if (transportClientMode) { - return Collections.emptyList(); - } Set headers = new HashSet<>(); headers.add(UsernamePasswordToken.BASIC_AUTH_HEADER); if (XPackSettings.AUDIT_ENABLED.get(settings)) { @@ -773,11 +751,7 @@ public List getActionFilters() { if (enabled == false) { return emptyList(); } - // registering the security filter only for nodes - if (transportClientMode == false) { - return singletonList(securityActionFilter.get()); - } - return emptyList(); + return singletonList(securityActionFilter.get()); } @Override @@ -865,7 +839,7 @@ static void validateRealmSettings(Settings settings) { @Override public List getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) { - if (transportClientMode || enabled == false) { // don't register anything if we are not enabled + if (enabled == false) { // don't register anything if we are not enabled // interceptors are not installed if we are running on the transport client return Collections.emptyList(); } @@ -890,7 +864,7 @@ public AsyncSender interceptSender(AsyncSender sender) { public Map> getTransports(Settings settings, ThreadPool threadPool, PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService) { - if (transportClientMode || enabled == false) { // don't register anything if we are not enabled, or in transport client mode + if (enabled == false) { // don't register anything if we are not enabled, or in transport client mode return Collections.emptyMap(); } @@ -944,7 +918,7 @@ public Map> getHttpTransports(Settings set @Override public UnaryOperator getRestHandlerWrapper(ThreadContext threadContext) { - if (enabled == false || transportClientMode) { + if (enabled == false) { return null; } final boolean ssl = HTTP_SSL_ENABLED.get(settings); @@ -955,7 +929,7 @@ public UnaryOperator getRestHandlerWrapper(ThreadContext threadCont @Override public List> getExecutorBuilders(final Settings settings) { - if (enabled && transportClientMode == false) { + if (enabled) { return Collections.singletonList( new FixedExecutorBuilder(settings, TokenService.THREAD_POOL_NAME, 1, 1000, "xpack.security.authc.token.thread_pool")); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/AbstractSecurityModule.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/AbstractSecurityModule.java deleted file mode 100644 index 0dfb369bc371f..0000000000000 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/AbstractSecurityModule.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security.support; - -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.core.XPackSettings; - -public abstract class AbstractSecurityModule extends AbstractModule { - - protected final Settings settings; - protected final boolean clientMode; - protected final boolean securityEnabled; - - public AbstractSecurityModule(Settings settings) { - this.settings = settings; - this.clientMode = TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey())); - this.securityEnabled = XPackSettings.SECURITY_ENABLED.get(settings); - } - - @Override - protected final void configure() { - configure(clientMode); - } - - protected abstract void configure(boolean clientMode); - - public abstract static class Node extends AbstractSecurityModule { - - protected Node(Settings settings) { - super(settings); - } - - @Override - protected final void configure(boolean clientMode) { - assert !clientMode : "[" + getClass().getSimpleName() + "] is a node only module"; - configureNode(); - } - - protected abstract void configureNode(); - } -} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java index bb0036e9f870a..db5a22c5e6e9d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java @@ -42,7 +42,7 @@ public Settings nodeSettings(int nodeOrdinal) { public void testThatBulkUpdateDoesNotLoseFields() { assertEquals(DocWriteResponse.Result.CREATED, client().prepareIndex("index1", "type").setSource("{\"test\": \"test\"}", XContentType.JSON).setId("1").get().getResult()); - GetResponse getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").get(); + GetResponse getResponse = client().prepareGet("index1", "type", "1").get(); assertEquals("test", getResponse.getSource().get("test")); if (randomBoolean()) { @@ -50,9 +50,9 @@ public void testThatBulkUpdateDoesNotLoseFields() { } // update with a new field - assertEquals(DocWriteResponse.Result.UPDATED, internalCluster().transportClient().prepareUpdate("index1", "type", "1") + assertEquals(DocWriteResponse.Result.UPDATED, client().prepareUpdate("index1", "type", "1") .setDoc("{\"not test\": \"not test\"}", XContentType.JSON).get().getResult()); - getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").get(); + getResponse = client().prepareGet("index1", "type", "1").get(); assertEquals("test", getResponse.getSource().get("test")); assertEquals("not test", getResponse.getSource().get("not test")); @@ -61,10 +61,10 @@ public void testThatBulkUpdateDoesNotLoseFields() { flushAndRefresh(); // do it in a bulk - BulkResponse response = internalCluster().transportClient().prepareBulk().add(client().prepareUpdate("index1", "type", "1") + BulkResponse response = client().prepareBulk().add(client().prepareUpdate("index1", "type", "1") .setDoc("{\"bulk updated\": \"bulk updated\"}", XContentType.JSON)).get(); assertEquals(DocWriteResponse.Result.UPDATED, response.getItems()[0].getResponse().getResult()); - getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").get(); + getResponse = client().prepareGet("index1", "type", "1").get(); assertEquals("test", getResponse.getSource().get("test")); assertEquals("not test", getResponse.getSource().get("not test")); assertEquals("bulk updated", getResponse.getSource().get("bulk updated")); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java index 6d7eacfe26cfa..2a9f2017c50f3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java @@ -74,7 +74,7 @@ protected boolean addMockHttpTransport() { } public void testModifyingViaApiClearsCache() throws Exception { - Client client = internalCluster().transportClient(); + Client client = client(); SecurityClient securityClient = securityClient(client); int modifiedRolesCount = randomIntBetween(1, roles.length); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 3055d1b0f456b..5e83ef99563d9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -64,7 +64,7 @@ import static org.hamcrest.Matchers.nullValue; // The random usage of meta fields such as _timestamp add noise to the test, so disable random index templates: -@ESIntegTestCase.ClusterScope +@ESIntegTestCase.ClusterScope(transportClientRatio = 0.0) public class FieldLevelSecurityTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java index da03e9ffe3d1e..2035a8b6c19dd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java @@ -125,7 +125,7 @@ public void testSingleRole() throws Exception { refresh(); - Client client = internalCluster().transportClient(); + Client client = client(); // no specifying an index, should replace indices with the permitted ones (test & test1) SearchResponse searchResponse = client.prepareSearch().setQuery(matchAllQuery()).get(); @@ -246,7 +246,7 @@ public void testMultipleRoles() throws Exception { refresh(); - Client client = internalCluster().transportClient(); + Client client = client(); SearchResponse response = client .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_a", USERS_PASSWD))) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java index c07491dc86314..fb85061110e08 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java @@ -72,24 +72,14 @@ protected SecureString nodeClientPassword() { return new SecureString("test123".toCharArray()); } - @Override - protected String transportClientUsername() { - return "admin"; - } - - @Override - protected SecureString transportClientPassword() { - return new SecureString("test123".toCharArray()); - } - public void testDifferentCombinationsOfIndices() throws Exception { - Client client = internalCluster().transportClient(); + Client client = client(); // first lets try with "admin"... all should work AcknowledgedResponse putResponse = client .filterWithHeader(Collections.singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, - basicAuthHeaderValue(transportClientUsername(), transportClientPassword()))) + basicAuthHeaderValue(nodeClientUsername(), nodeClientPassword()))) .admin().indices().preparePutTemplate("template1") .setPatterns(Collections.singletonList("test_*")) .get(); @@ -103,7 +93,7 @@ public void testDifferentCombinationsOfIndices() throws Exception { // now lets try with "user" Map auth = Collections.singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue("user", - transportClientPassword())); + nodeClientPassword())); assertThrowsAuthorizationException(client.filterWithHeader(auth).admin().indices().preparePutTemplate("template1") .setPatterns(Collections.singletonList("test_*"))::get, PutIndexTemplateAction.NAME, "user"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java index 9f86887566ac4..4c189e3e7f3da 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java @@ -93,7 +93,7 @@ public void testThatClearingAllScrollIdsWorks() throws Exception { Map headers = new HashMap<>(); headers.put(SecurityField.USER_SETTING.getKey(), user); headers.put(BASIC_AUTH_HEADER, basicAuth); - ClearScrollResponse clearScrollResponse = internalCluster().transportClient().filterWithHeader(headers) + ClearScrollResponse clearScrollResponse = client().filterWithHeader(headers) .prepareClearScroll() .addScrollId("_all").get(); assertThat(clearScrollResponse.isSucceeded(), is(true)); @@ -107,7 +107,7 @@ public void testThatClearingAllScrollIdsRequirePermissions() throws Exception { Map headers = new HashMap<>(); headers.put(SecurityField.USER_SETTING.getKey(), user); headers.put(BASIC_AUTH_HEADER, basicAuth); - assertThrows(internalCluster().transportClient().filterWithHeader(headers) + assertThrows(client().filterWithHeader(headers) .prepareClearScroll() .addScrollId("_all"), ElasticsearchSecurityException.class, "action [cluster:admin/indices/scroll/clear_all] is unauthorized for user [denied_user]"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java index 87db72bcf0285..349bef3fc3152 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java @@ -18,7 +18,7 @@ /** * Integration test that uses multiple data nodes to test that the shrink index api works with security. */ -@ClusterScope(minNumDataNodes = 2) +@ClusterScope(minNumDataNodes = 2, transportClientRatio = 0.0) public class ShrinkIndexWithSecurityTests extends SecurityIntegTestCase { @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index 671a94452fa0a..78d95ecbca0b8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -46,7 +46,7 @@ public void stopESNativeStores() throws Exception { if (getCurrentClusterScope() == Scope.SUITE) { // Clear the realm cache for all realms since we use a SUITE scoped cluster - SecurityClient client = securityClient(internalCluster().transportClient()); + SecurityClient client = securityClient(client()); client.prepareClearRealmCache().get(); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 462e4e26541e6..d862d248976da 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -7,7 +7,6 @@ import io.netty.util.ThreadDeathWatcher; import io.netty.util.concurrent.GlobalEventExecutor; - import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; @@ -26,7 +25,6 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkAddress; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -40,7 +38,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.core.XPackClient; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; @@ -77,6 +74,7 @@ * * @see SecuritySettingsSource */ +@ESIntegTestCase.ClusterScope(transportClientRatio = 0.0) public abstract class SecurityIntegTestCase extends ESIntegTestCase { private static SecuritySettingsSource SECURITY_DEFAULT_SETTINGS; @@ -260,14 +258,6 @@ protected Path nodeConfigPath(int nodeOrdinal) { return customSecuritySettingsSource.nodeConfigPath(nodeOrdinal); } - @Override - protected Settings transportClientSettings() { - return Settings.builder().put(super.transportClientSettings()) - .put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NIO) - .put(customSecuritySettingsSource.transportClientSettings()) - .build(); - } - @Override protected boolean addMockTransportService() { return false; // security has its own transport service @@ -278,19 +268,6 @@ protected Collection> nodePlugins() { return customSecuritySettingsSource.nodePlugins(); } - @Override - protected Collection> transportClientPlugins() { - return customSecuritySettingsSource.transportClientPlugins(); - } - - @Override - protected Settings externalClusterClientSettings() { - return Settings.builder() - .put(SecurityField.USER_SETTING.getKey(), SecuritySettingsSource.TEST_USER_NAME + ":" - + SecuritySettingsSourceField.TEST_PASSWORD) - .build(); - } - /** * Allows to override the users config file when the {@link org.elasticsearch.test.ESIntegTestCase.ClusterScope} is set to * {@link org.elasticsearch.test.ESIntegTestCase.Scope#SUITE} or {@link org.elasticsearch.test.ESIntegTestCase.Scope#TEST} @@ -333,24 +310,6 @@ protected SecureString nodeClientPassword() { return SECURITY_DEFAULT_SETTINGS.nodeClientPassword(); } - /** - * Allows to override the transport client username (used while sending requests to the test cluster) when the - * {@link org.elasticsearch.test.ESIntegTestCase.ClusterScope} is set to - * {@link org.elasticsearch.test.ESIntegTestCase.Scope#SUITE} or {@link org.elasticsearch.test.ESIntegTestCase.Scope#TEST} - */ - protected String transportClientUsername() { - return SECURITY_DEFAULT_SETTINGS.transportClientUsername(); - } - - /** - * Allows to override the transport client password (used while sending requests to the test cluster) when the - * {@link org.elasticsearch.test.ESIntegTestCase.ClusterScope} is set to - * {@link org.elasticsearch.test.ESIntegTestCase.Scope#SUITE} or {@link org.elasticsearch.test.ESIntegTestCase.Scope#TEST} - */ - protected SecureString transportClientPassword() { - return SECURITY_DEFAULT_SETTINGS.transportClientPassword(); - } - /** * Allows to control whether ssl key information is auto generated or not on the transport layer */ @@ -392,16 +351,6 @@ protected String nodeClientUsername() { protected SecureString nodeClientPassword() { return SecurityIntegTestCase.this.nodeClientPassword(); } - - @Override - protected String transportClientUsername() { - return SecurityIntegTestCase.this.transportClientUsername(); - } - - @Override - protected SecureString transportClientPassword() { - return SecurityIntegTestCase.this.transportClientPassword(); - } } protected static void assertGreenClusterState(Client client) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 8e6e00f32a90e..768bc38813c0b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -118,8 +118,8 @@ protected SSLService getSslService() { }; ThreadPool threadPool = mock(ThreadPool.class); ClusterService clusterService = mock(ClusterService.class); - settings = Security.additionalSettings(settings, true, false); - Set> allowedSettings = new HashSet<>(Security.getSettings(false, null)); + settings = Security.additionalSettings(settings, true); + Set> allowedSettings = new HashSet<>(Security.getSettings(null)); allowedSettings.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterSettings clusterSettings = new ClusterSettings(settings, allowedSettings); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); @@ -180,7 +180,7 @@ public void testDisabledByDefault() throws Exception { } public void testHttpSettingDefaults() throws Exception { - final Settings defaultSettings = Security.additionalSettings(Settings.EMPTY, true, false); + final Settings defaultSettings = Security.additionalSettings(Settings.EMPTY, true); assertThat(SecurityField.NAME4, equalTo(NetworkModule.TRANSPORT_TYPE_SETTING.get(defaultSettings))); assertThat(SecurityField.NAME4, equalTo(NetworkModule.HTTP_TYPE_SETTING.get(defaultSettings))); } @@ -189,7 +189,7 @@ public void testTransportSettingNetty4Both() { Settings both4 = Security.additionalSettings(Settings.builder() .put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4) .put(NetworkModule.HTTP_TYPE_KEY, SecurityField.NAME4) - .build(), true, false); + .build(), true); assertFalse(NetworkModule.TRANSPORT_TYPE_SETTING.exists(both4)); assertFalse(NetworkModule.HTTP_TYPE_SETTING.exists(both4)); } @@ -198,13 +198,13 @@ public void testTransportSettingValidation() { final String badType = randomFrom("netty4", "other", "security1"); Settings settingsTransport = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, badType).build(); IllegalArgumentException badTransport = expectThrows(IllegalArgumentException.class, - () -> Security.additionalSettings(settingsTransport, true, false)); + () -> Security.additionalSettings(settingsTransport, true)); assertThat(badTransport.getMessage(), containsString(SecurityField.NAME4)); assertThat(badTransport.getMessage(), containsString(NetworkModule.TRANSPORT_TYPE_KEY)); Settings settingsHttp = Settings.builder().put(NetworkModule.HTTP_TYPE_KEY, badType).build(); IllegalArgumentException badHttp = expectThrows(IllegalArgumentException.class, - () -> Security.additionalSettings(settingsHttp, true, false)); + () -> Security.additionalSettings(settingsHttp, true)); assertThat(badHttp.getMessage(), containsString(SecurityField.NAME4)); assertThat(badHttp.getMessage(), containsString(NetworkModule.HTTP_TYPE_KEY)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java index f6e5552ddbc53..b04b8c8ac3d36 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java @@ -32,7 +32,7 @@ * templates when started within security, as this requires certain * system privileges */ -@ClusterScope(maxNumDataNodes = 1, scope = Scope.SUITE, numClientNodes = 0) +@ClusterScope(maxNumDataNodes = 1, scope = Scope.SUITE, numClientNodes = 0, transportClientRatio = 0.0) public class TemplateUpgraderTests extends SecurityIntegTestCase { public void testTemplatesWorkAsExpected() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java index 23408f5668ec9..866c52989af6f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java @@ -29,7 +29,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@ClusterScope(scope = TEST, numDataNodes = 1) +@ClusterScope(scope = TEST, numDataNodes = 1, transportClientRatio = 0.0) public class AuditTrailSettingsUpdateTests extends SecurityIntegTestCase { private static Settings startupFilterSettings; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java index f507edf97874f..3290aba27e37f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchContextMissingException; import org.elasticsearch.test.SecurityIntegTestCase; @@ -99,14 +98,4 @@ public void testSearchAndClearScroll() throws Exception { public void cleanupSecurityIndex() throws Exception { super.deleteSecurityIndex(); } - - @Override - public String transportClientUsername() { - return this.nodeClientUsername(); - } - - @Override - public SecureString transportClientPassword() { - return this.nodeClientPassword(); - } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java index bc17626b1f426..9f0b7863d30e7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.security.transport.filter; -import org.elasticsearch.client.Client; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -61,11 +60,6 @@ public void testThatIpFilteringIsIntegratedIntoNettyPipelineViaHttp() throws Exc } } - public void testThatIpFilteringIsNotAppliedForDefaultTransport() throws Exception { - Client client = internalCluster().transportClient(); - assertGreenClusterState(client); - } - public void testThatIpFilteringIsAppliedForProfile() throws Exception { try (Socket socket = new Socket()){ trySocketConnection(socket, getProfileAddress("client")); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java index 65a5fb080cdb0..96922aa8822e4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java @@ -21,7 +21,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.is; -@ClusterScope(scope = TEST, supportsDedicatedMasters = false, numDataNodes = 1) +@ClusterScope(scope = TEST, supportsDedicatedMasters = false, numDataNodes = 1, transportClientRatio = 0.0) public class IpFilteringUpdateTests extends SecurityIntegTestCase { private static int randomClientPort; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java deleted file mode 100644 index fe1b65e851d0f..0000000000000 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security.transport.netty4; - -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.transport.TransportSettings; -import org.elasticsearch.xpack.core.ssl.SSLClientAuth; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; -import static org.hamcrest.CoreMatchers.is; - -// TODO delete this test? -public class IPHostnameVerificationTests extends SecurityIntegTestCase { - private Path certPath; - private Path keyPath; - - @Override - protected boolean transportSSLEnabled() { - return true; - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings settings = super.nodeSettings(nodeOrdinal); - Settings.Builder builder = Settings.builder() - .put(settings.filter((s) -> s.startsWith("xpack.security.transport.ssl.") == false), false); - settings = builder.build(); - - // The default Unicast test behavior is to use 'localhost' with the port number. For this test we need to use IP - List newUnicastAddresses = new ArrayList<>(); - for (String address : settings.getAsList(DISCOVERY_SEED_HOSTS_SETTING.getKey())) { - newUnicastAddresses.add(address.replace("localhost", "127.0.0.1")); - } - - Settings.Builder settingsBuilder = Settings.builder() - .put(settings) - .putList(DISCOVERY_SEED_HOSTS_SETTING.getKey(), newUnicastAddresses); - - try { - //Use a cert with a CN of "Elasticsearch Test Node" and IPv4+IPv6 ip addresses as SubjectAlternativeNames - certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.crt"); - keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.pem"); - assertThat(Files.exists(certPath), is(true)); - } catch (Exception e) { - throw new RuntimeException(e); - } - - SecuritySettingsSource.addSecureSettings(settingsBuilder, secureSettings -> { - secureSettings.setString("xpack.security.transport.ssl.secure_key_passphrase", "testnode-ip-only"); - }); - return settingsBuilder.put("xpack.security.transport.ssl.key", keyPath.toAbsolutePath()) - .put("xpack.security.transport.ssl.certificate", certPath.toAbsolutePath()) - .put("xpack.security.transport.ssl.certificate_authorities", certPath.toAbsolutePath()) - .put(TransportSettings.BIND_HOST.getKey(), "127.0.0.1") - .put("network.host", "127.0.0.1") - .put("xpack.security.transport.ssl.client_authentication", SSLClientAuth.NONE) - .put("xpack.security.transport.ssl.verification_mode", "full") - .build(); - } - - @Override - protected Settings transportClientSettings() { - Settings clientSettings = super.transportClientSettings(); - return Settings.builder().put(clientSettings.filter(k -> k.startsWith("xpack.security.transport.ssl.") == false)) - .put("xpack.security.transport.ssl.verification_mode", "certificate") - .put("xpack.security.transport.ssl.key", keyPath.toAbsolutePath()) - .put("xpack.security.transport.ssl.certificate", certPath.toAbsolutePath()) - .put("xpack.security.transport.ssl.key_passphrase", "testnode-ip-only") - .put("xpack.security.transport.ssl.certificate_authorities", certPath) - .build(); - } - - public void testTransportClientConnectionWorksWithIPOnlyHostnameVerification() throws Exception { - Client client = internalCluster().transportClient(); - assertGreenClusterState(client); - } -} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java index ce0cc5c111265..88d27d4171a19 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java @@ -119,7 +119,7 @@ public void testThatHttpWorksWithSslClientAuth() throws IOException { try (RestClient restClient = createRestClient(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy), "https")) { Request request = new Request("GET", "/"); RequestOptions.Builder options = request.getOptions().toBuilder(); - options.addHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword())); + options.addHeader("Authorization", basicAuthHeaderValue(nodeClientUsername(), nodeClientPassword())); request.setOptions(options); Response response = restClient.performRequest(request); assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java index 944c3306763a6..9c540f559b688 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java @@ -49,7 +49,7 @@ * * @see RestrictedTrustManager */ -@ESIntegTestCase.ClusterScope(numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) +@ESIntegTestCase.ClusterScope(numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false, transportClientRatio = 0.0) @TestLogging("org.elasticsearch.xpack.ssl.RestrictedTrustManager:DEBUG") public class SSLTrustRestrictionsTests extends SecurityIntegTestCase { @@ -149,15 +149,6 @@ private void writeRestrictions(String trustedPattern) { runResourceWatcher(); } - @Override - protected Settings transportClientSettings() { - Settings parentSettings = super.transportClientSettings(); - Settings.Builder builder = Settings.builder() - .put(parentSettings.filter((s) -> s.startsWith("xpack.security.transport.ssl.") == false)) - .put(nodeSSL); - return builder.build(); - } - @Override protected boolean transportSSLEnabled() { return true; diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index dd7f268e1a6f5..4226ff4fed0d9 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -226,16 +226,14 @@ public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin, Reloa private BulkProcessor bulkProcessor; protected final Settings settings; - protected final boolean transportClient; protected final boolean enabled; protected List reloadableServices = new ArrayList<>(); public Watcher(final Settings settings) { this.settings = settings; - this.transportClient = XPackPlugin.transportClientMode(settings); this.enabled = XPackSettings.WATCHER_ENABLED.get(settings); - if (enabled && transportClient == false) { + if (enabled) { validAutoCreateIndex(settings, logger); } } @@ -433,7 +431,7 @@ public Collection createGuiceModules() { modules.add(b -> b.bind(Clock.class).toInstance(getClock())); //currently assuming the only place clock is bound modules.add(b -> { XPackPlugin.bindFeatureSet(b, WatcherFeatureSet.class); - if (transportClient || enabled == false) { + if (enabled == false) { b.bind(WatcherService.class).toProvider(Providers.of(null)); } }); @@ -567,7 +565,7 @@ public List getRestHandlers(Settings settings, RestController restC @Override public void onIndexModule(IndexModule module) { - if (enabled == false || transportClient) { + if (enabled == false) { return; } @@ -676,7 +674,7 @@ public void close() throws IOException { */ @Override public void reload(Settings settings) { - if (enabled == false || transportClient) { + if (enabled == false) { return; } reloadableServices.forEach(s -> s.reload(settings)); diff --git a/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java b/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java index 0a75565fbc075..855162a0b86fc 100644 --- a/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java +++ b/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java @@ -5,91 +5,175 @@ */ package org.elasticsearch.xpack.security; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.reindex.DeleteByQueryAction; -import org.elasticsearch.index.reindex.DeleteByQueryRequestBuilder; -import org.elasticsearch.index.reindex.ReindexAction; -import org.elasticsearch.index.reindex.ReindexRequestBuilder; -import org.elasticsearch.index.reindex.UpdateByQueryAction; -import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; -import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.xpack.core.security.SecurityField; - - +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.util.Collections; + +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + + +public class ReindexWithSecurityIT extends ESRestTestCase { + + private static final String USER = "test_admin"; + private static final String PASS = "x-pack-test-password"; + + private static Path httpTrustStore; + + @BeforeClass + public static void findTrustStore( ) throws Exception { + final URL resource = ReindexWithSecurityClientYamlTestSuiteIT.class.getResource("/ssl/ca.p12"); + if (resource == null) { + throw new FileNotFoundException("Cannot find classpath resource /ssl/ca.p12"); + } + httpTrustStore = PathUtils.get(resource.toURI()); + } -public class ReindexWithSecurityIT extends SecurityIntegTestCase { + @AfterClass + public static void cleanupStatics() { + httpTrustStore = null; + } @Override - protected Settings externalClusterClientSettings() { - Settings.Builder builder = Settings.builder().put(super.externalClusterClientSettings()); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4); - builder.put(SecurityField.USER_SETTING.getKey(), "test_admin:x-pack-test-password"); - return builder.build(); + protected String getProtocol() { + return "https"; } /** - * TODO: this entire class should be removed. SecurityIntegTestCase is meant for tests, but we run against real xpack + * All tests run as a an administrative user but use es-security-runas-user to become a less privileged user. */ @Override - public void doAssertXPackIsInstalled() { - // this assertion doesn't make sense with a real distribution, since there is not currently a way - // from nodes info to see which modules are loaded + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .put(TRUSTSTORE_PATH , httpTrustStore) + .put(TRUSTSTORE_PASSWORD, "password") + .build(); } - public void testDeleteByQuery() { + public void testDeleteByQuery() throws IOException { createIndicesWithRandomAliases("test1", "test2", "test3"); - BulkByScrollResponse response = new DeleteByQueryRequestBuilder(client(), DeleteByQueryAction.INSTANCE) - .source("test1", "test2") - .filter(QueryBuilders.matchAllQuery()) - .get(); + RestHighLevelClient restClient = new TestRestHighLevelClient(); + BulkByScrollResponse response = restClient.deleteByQuery((DeleteByQueryRequest) new DeleteByQueryRequest() + .setQuery(QueryBuilders.matchAllQuery()) + .indices("test1", "test2"), RequestOptions.DEFAULT); assertNotNull(response); - response = new DeleteByQueryRequestBuilder(client(), DeleteByQueryAction.INSTANCE) - .source("test*") - .filter(QueryBuilders.matchAllQuery()) - .get(); + response = restClient.deleteByQuery((DeleteByQueryRequest) new DeleteByQueryRequest() + .setQuery(QueryBuilders.matchAllQuery()) + .indices("test*"), RequestOptions.DEFAULT); assertNotNull(response); - IndexNotFoundException e = expectThrows(IndexNotFoundException.class, - () -> new DeleteByQueryRequestBuilder(client(), DeleteByQueryAction.INSTANCE) - .source("test1", "index1") - .filter(QueryBuilders.matchAllQuery()) - .get()); - assertEquals("no such index [index1]", e.getMessage()); + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, + () -> restClient.deleteByQuery((DeleteByQueryRequest) new DeleteByQueryRequest() + .setQuery(QueryBuilders.matchAllQuery()) + .indices("test1", "index1"), RequestOptions.DEFAULT)); + assertThat(e.getMessage(), containsString("no such index [index1]")); } - public void testUpdateByQuery() { + public void testUpdateByQuery() throws IOException { createIndicesWithRandomAliases("test1", "test2", "test3"); - BulkByScrollResponse response = new UpdateByQueryRequestBuilder(client(), UpdateByQueryAction.INSTANCE) - .source("test1", "test2").get(); + RestHighLevelClient restClient = new TestRestHighLevelClient(); + BulkByScrollResponse response = + restClient.updateByQuery((UpdateByQueryRequest) new UpdateByQueryRequest().indices("test1", "test2"), RequestOptions.DEFAULT); assertNotNull(response); - response = new UpdateByQueryRequestBuilder(client(), UpdateByQueryAction.INSTANCE).source("test*").get(); + response = restClient.updateByQuery((UpdateByQueryRequest) new UpdateByQueryRequest().indices("test*"), RequestOptions.DEFAULT); assertNotNull(response); - IndexNotFoundException e = expectThrows(IndexNotFoundException.class, - () -> new UpdateByQueryRequestBuilder(client(), UpdateByQueryAction.INSTANCE).source("test1", "index1").get()); - assertEquals("no such index [index1]", e.getMessage()); + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, + () -> restClient.updateByQuery((UpdateByQueryRequest) new UpdateByQueryRequest().indices("test1", "index1"), + RequestOptions.DEFAULT)); + assertThat(e.getMessage(), containsString("no such index [index1]")); } - public void testReindex() { + public void testReindex() throws IOException { createIndicesWithRandomAliases("test1", "test2", "test3", "dest"); - BulkByScrollResponse response = new ReindexRequestBuilder(client(), ReindexAction.INSTANCE).source("test1", "test2") - .destination("dest").get(); + RestHighLevelClient restClient = new TestRestHighLevelClient(); + BulkByScrollResponse response = restClient.reindex(new ReindexRequest().setSourceIndices("test1", "test2").setDestIndex("dest"), + RequestOptions.DEFAULT); assertNotNull(response); - response = new ReindexRequestBuilder(client(), ReindexAction.INSTANCE).source("test*").destination("dest").get(); + response = restClient.reindex(new ReindexRequest().setSourceIndices("test*").setDestIndex("dest"), + RequestOptions.DEFAULT); assertNotNull(response); - IndexNotFoundException e = expectThrows(IndexNotFoundException.class, - () -> new ReindexRequestBuilder(client(), ReindexAction.INSTANCE).source("test1", "index1").destination("dest").get()); - assertEquals("no such index [index1]", e.getMessage()); + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, + () -> restClient.reindex(new ReindexRequest().setSourceIndices("test1", "index1").setDestIndex("dest"), + RequestOptions.DEFAULT)); + assertThat(e.getMessage(), containsString("no such index [index1]")); + } + + /** + * Creates the indices provided as argument, randomly associating them with aliases, indexes one dummy document per index + * and refreshes the new indices + */ + private void createIndicesWithRandomAliases(String... indices) throws IOException { + for (String index : indices) { + createIndex(index, Settings.EMPTY); + } + + RestHighLevelClient restClient = new TestRestHighLevelClient(); + if (frequently()) { + boolean aliasAdded = false; + + IndicesAliasesRequest request = new IndicesAliasesRequest(); + for (String index : indices) { + if (frequently()) { + //one alias per index with prefix "alias-" + request.addAliasAction(AliasActions.add().index(index).alias("alias-" + index)); + aliasAdded = true; + } + } + // If we get to this point and we haven't added an alias to the request we need to add one + // or the request will fail so use noAliasAdded to force adding the alias in this case + if (aliasAdded == false || randomBoolean()) { + //one alias pointing to all indices + for (String index : indices) { + request.addAliasAction(AliasActions.add().index(index).alias("alias")); + } + } + AcknowledgedResponse response = restClient.indices().updateAliases(request, RequestOptions.DEFAULT); + assertThat(response.isAcknowledged(), is(true)); + } + + for (String index : indices) { + restClient.index(new IndexRequest(index).source("field", "value"), RequestOptions.DEFAULT); + } + restClient.indices().refresh(new RefreshRequest(indices), RequestOptions.DEFAULT); + } + + private class TestRestHighLevelClient extends RestHighLevelClient { + TestRestHighLevelClient() { + super(client(), restClient -> {}, Collections.emptyList()); + } } } diff --git a/x-pack/qa/security-client-tests/build.gradle b/x-pack/qa/security-client-tests/build.gradle deleted file mode 100644 index 556e36e51467f..0000000000000 --- a/x-pack/qa/security-client-tests/build.gradle +++ /dev/null @@ -1,40 +0,0 @@ -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' - -dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" - testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') -} - -String outputDir = "${buildDir}/generated-resources/${project.name}" -task copyXPackPluginProps(type: Copy) { - from project(xpackModule('core')).file('src/main/plugin-metadata') - from project(xpackModule('core')).tasks.pluginProperties - into outputDir -} -project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps) - -integTestRunner { - systemProperty 'tests.security.manager', 'false' -} - -integTestCluster { - setting 'xpack.ilm.enabled', 'false' - setting 'xpack.security.enabled', 'true' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.license.self_generated.type', 'trial' - setupCommand 'setupDummyUser', - 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser' - setupCommand 'setupTransportClientUser', - 'bin/elasticsearch-users', 'useradd', 'transport', '-p', 'x-pack-test-password', '-r', 'transport_client' - waitCondition = { node, ant -> - File tmpFile = new File(node.cwd, 'wait.success') - ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow", - dest: tmpFile.toString(), - username: 'test_user', - password: 'x-pack-test-password', - ignoreerrors: true, - retries: 10) - return tmpFile.exists() - } -} diff --git a/x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java b/x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java deleted file mode 100644 index 519f365d515a0..0000000000000 --- a/x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security.qa; - -import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.XPackClientPlugin; -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -import org.elasticsearch.xpack.core.security.SecurityField; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.is; - -/** - * Integration tests that test a transport client with security being loaded that connect to an external cluster - */ -public class SecurityTransportClientIT extends ESIntegTestCase { - static final String ADMIN_USER_PW = "test_user:x-pack-test-password"; - static final String TRANSPORT_USER_PW = "transport:x-pack-test-password"; - - @Override - protected Settings externalClusterClientSettings() { - return Settings.builder() - .put(SecurityField.USER_SETTING.getKey(), ADMIN_USER_PW) - .put(NetworkModule.TRANSPORT_TYPE_KEY, "security4") - .build(); - } - - @Override - protected Collection> transportClientPlugins() { - return Collections.singletonList(XPackClientPlugin.class); - } - - public void testThatTransportClientWithoutAuthenticationDoesNotWork() throws Exception { - try (TransportClient client = transportClient(Settings.EMPTY)) { - boolean connected = awaitBusy(() -> { - return client.connectedNodes().size() > 0; - }, 5L, TimeUnit.SECONDS); - - assertThat(connected, is(false)); - } - } - - public void testThatTransportClientAuthenticationWithTransportClientRole() throws Exception { - Settings settings = Settings.builder() - .put(SecurityField.USER_SETTING.getKey(), TRANSPORT_USER_PW) - .build(); - try (TransportClient client = transportClient(settings)) { - boolean connected = awaitBusy(() -> { - return client.connectedNodes().size() > 0; - }, 5L, TimeUnit.SECONDS); - - assertThat(connected, is(true)); - - // this checks that the transport client is really running in a limited state - try { - client.admin().cluster().prepareHealth().get(); - fail("the transport user should not be be able to get health!"); - } catch (ElasticsearchSecurityException e) { - assertThat(e.toString(), containsString("unauthorized")); - } - } - } - - public void testTransportClientWithAdminUser() throws Exception { - final boolean useTransportUser = randomBoolean(); - Settings settings = Settings.builder() - .put(SecurityField.USER_SETTING.getKey(), useTransportUser ? TRANSPORT_USER_PW : ADMIN_USER_PW) - .build(); - try (TransportClient client = transportClient(settings)) { - boolean connected = awaitBusy(() -> { - return client.connectedNodes().size() > 0; - }, 5L, TimeUnit.SECONDS); - - assertThat(connected, is(true)); - - // this checks that the transport client is really running in a limited state - ClusterHealthResponse response; - if (useTransportUser) { - response = client.filterWithHeader(Collections.singletonMap("Authorization", - basicAuthHeaderValue("test_user", new SecureString("x-pack-test-password".toCharArray())))) - .admin().cluster().prepareHealth().get(); - } else { - response = client.admin().cluster().prepareHealth().get(); - } - - assertThat(response.isTimedOut(), is(false)); - } - } - - TransportClient transportClient(Settings extraSettings) { - NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get(); - List nodes = nodeInfos.getNodes(); - assertTrue(nodes.isEmpty() == false); - TransportAddress publishAddress = randomFrom(nodes).getTransport().address().publishAddress(); - String clusterName = nodeInfos.getClusterName().value(); - - Settings settings = Settings.builder() - .put(extraSettings) - .put("cluster.name", clusterName) - .build(); - - TransportClient client = new PreBuiltXPackTransportClient(settings); - client.addTransportAddress(publishAddress); - return client; - } -} diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle index 1ff65519c367d..4790df3609c35 100644 --- a/x-pack/qa/security-example-spi-extension/build.gradle +++ b/x-pack/qa/security-example-spi-extension/build.gradle @@ -9,7 +9,7 @@ esplugin { dependencies { compileOnly "org.elasticsearch.plugin:x-pack-core:${version}" - testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') + testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}" } diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java index 4487187a80b6d..e75c7705ef9bf 100644 --- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java +++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java @@ -5,60 +5,43 @@ */ package org.elasticsearch.example.realm; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.env.Environment; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -import org.elasticsearch.xpack.core.XPackClientPlugin; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; +import org.elasticsearch.test.rest.ESRestTestCase; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; /** * Integration test to test authentication with the custom realm */ -public class CustomRealmIT extends ESIntegTestCase { +public class CustomRealmIT extends ESRestTestCase { @Override - protected Settings externalClusterClientSettings() { + protected Settings restClientSettings() { return Settings.builder() .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER) .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()) - .put(NetworkModule.TRANSPORT_TYPE_KEY, "security4") .build(); } - @Override - protected Collection> transportClientPlugins() { - return Collections.>singleton(XPackClientPlugin.class); - } - - public void testHttpConnectionWithNoAuthentication() throws Exception { - try { - getRestClient().performRequest(new Request("GET", "/")); - fail("request should have failed"); - } catch(ResponseException e) { - Response response = e.getResponse(); - assertThat(response.getStatusLine().getStatusCode(), is(401)); - String value = response.getHeader("WWW-Authenticate"); - assertThat(value, is("custom-challenge")); - } + public void testHttpConnectionWithNoAuthentication() { + Request request = new Request("GET", "/"); + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.addHeader(CustomRealm.USER_HEADER, ""); + builder.addHeader(CustomRealm.PW_HEADER, ""); + request.setOptions(builder); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); + Response response = e.getResponse(); + assertThat(response.getStatusLine().getStatusCode(), is(401)); + String value = response.getHeader("WWW-Authenticate"); + assertThat(value, is("custom-challenge")); } public void testHttpAuthentication() throws Exception { @@ -67,59 +50,16 @@ public void testHttpAuthentication() throws Exception { options.addHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER); options.addHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()); request.setOptions(options); - Response response = getRestClient().performRequest(request); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } - public void testTransportClient() throws Exception { - NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get(); - List nodes = nodeInfos.getNodes(); - assertTrue(nodes.isEmpty() == false); - TransportAddress publishAddress = randomFrom(nodes).getTransport().address().publishAddress(); - String clusterName = nodeInfos.getClusterName().value(); - - Settings settings = Settings.builder() - .put("cluster.name", clusterName) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) - .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER) - .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()) - .build(); - try (TransportClient client = new PreBuiltXPackTransportClient(settings)) { - client.addTransportAddress(publishAddress); - ClusterHealthResponse response = client.admin().cluster().prepareHealth().execute().actionGet(); - assertThat(response.isTimedOut(), is(false)); - } - } - - public void testTransportClientWrongAuthentication() throws Exception { - NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get(); - List nodes = nodeInfos.getNodes(); - assertTrue(nodes.isEmpty() == false); - TransportAddress publishAddress = randomFrom(nodes).getTransport().address().publishAddress(); - String clusterName = nodeInfos.getClusterName().value(); - - Settings settings = Settings.builder() - .put("cluster.name", clusterName) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) - .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER + randomAlphaOfLength(1)) - .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()) - .build(); - try (TransportClient client = new PreBuiltXPackTransportClient(settings)) { - client.addTransportAddress(publishAddress); - client.admin().cluster().prepareHealth().execute().actionGet(); - fail("authentication failure should have resulted in a NoNodesAvailableException"); - } catch (NoNodeAvailableException e) { - // expected - } - } - public void testSettingsFiltering() throws Exception { - NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().clear().setSettings(true).get(); - for(NodeInfo info : nodeInfos.getNodes()) { - Settings settings = info.getSettings(); - assertNotNull(settings); - assertNull(settings.get("xpack.security.authc.realms.custom.custom.filtered_setting")); - assertEquals("0", settings.get("xpack.security.authc.realms.custom.custom.order")); - } + Request request = new Request("GET", "/_nodes/_all/settings"); + request.addParameter("flat_settings", "true"); + Response response = client().performRequest(request); + String responseString = EntityUtils.toString(response.getEntity()); + assertThat(responseString, not(containsString("xpack.security.authc.realms.custom.custom.filtered_setting"))); + assertThat(responseString, containsString("xpack.security.authc.realms.custom.custom.order")); } } diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java index 57a895848e3a8..3aab2a36562de 100644 --- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java +++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java @@ -9,20 +9,20 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.security.PutUserRequest; +import org.elasticsearch.client.security.RefreshPolicy; +import org.elasticsearch.client.security.user.User; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.example.realm.CustomRealm; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.XPackClientPlugin; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.client.SecurityClient; -import java.util.Collection; +import java.io.IOException; import java.util.Collections; +import java.util.List; import static org.elasticsearch.example.role.CustomInMemoryRolesProvider.INDEX; import static org.elasticsearch.example.role.CustomInMemoryRolesProvider.ROLE_A; @@ -33,7 +33,7 @@ /** * Integration test for custom roles providers. */ -public class CustomRolesProviderIT extends ESIntegTestCase { +public class CustomRolesProviderIT extends ESRestTestCase { private static final String TEST_USER = "test_user"; private static final String TEST_PWD = "change_me"; @@ -46,22 +46,17 @@ public class CustomRolesProviderIT extends ESIntegTestCase { } @Override - protected Settings externalClusterClientSettings() { + protected Settings restClientSettings() { return Settings.builder() - .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER) - .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()) - .put(NetworkModule.TRANSPORT_TYPE_KEY, "security4") - .build(); + .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER) + .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()) + .build(); } - @Override - protected Collection> transportClientPlugins() { - return Collections.singleton(XPackClientPlugin.class); - } - - public void setupTestUser(String role) { - SecurityClient securityClient = new SecurityClient(client()); - securityClient.preparePutUser(TEST_USER, TEST_PWD.toCharArray(), Hasher.BCRYPT, role).get(); + public void setupTestUser(String role) throws IOException { + new TestRestHighLevelClient().security().putUser( + PutUserRequest.withPassword(new User(TEST_USER, List.of(role)), TEST_PWD.toCharArray(), true, RefreshPolicy.IMMEDIATE), + RequestOptions.DEFAULT); } public void testAuthorizedCustomRoleSucceeds() throws Exception { @@ -69,7 +64,7 @@ public void testAuthorizedCustomRoleSucceeds() throws Exception { // roleB has all permissions on index "foo", so creating "foo" should succeed Request request = new Request("PUT", "/" + INDEX); request.setOptions(AUTH_OPTIONS); - Response response = getRestClient().performRequest(request); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -79,27 +74,23 @@ public void testFirstResolvedRoleTakesPrecedence() throws Exception { // the first custom role provider appears first in order, it should take precedence and deny // permission to create the index setupTestUser(ROLE_A); - // roleB has all permissions on index "foo", so creating "foo" should succeed - try { - Request request = new Request("PUT", "/" + INDEX); - request.setOptions(AUTH_OPTIONS); - getRestClient().performRequest(request); - fail(ROLE_A + " should not be authorized to create index " + INDEX); - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); - } + Request request = new Request("PUT", "/" + INDEX); + request.setOptions(AUTH_OPTIONS); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); } public void testUnresolvedRoleDoesntSucceed() throws Exception { setupTestUser("unknown"); - // roleB has all permissions on index "foo", so creating "foo" should succeed - try { - Request request = new Request("PUT", "/" + INDEX); - request.setOptions(AUTH_OPTIONS); - getRestClient().performRequest(request); - fail(ROLE_A + " should not be authorized to create index " + INDEX); - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); + Request request = new Request("PUT", "/" + INDEX); + request.setOptions(AUTH_OPTIONS); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); + } + + private class TestRestHighLevelClient extends RestHighLevelClient { + TestRestHighLevelClient() { + super(client(), restClient -> {}, Collections.emptyList()); } } } diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java index 2c2cdd044aab7..6ac2cdd3fb654 100644 --- a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java +++ b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java @@ -5,9 +5,9 @@ */ package org.elasticsearch.smoketest; -import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.rest.ESRestTestCase; -public class PreventFailingBuildIT extends ESIntegTestCase { +public class PreventFailingBuildIT extends ESRestTestCase { public void testSoThatTestsDoNotFail() { // Noop diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index e88eac3028f3d..d4fe2129363c5 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -7,6 +7,7 @@ apply plugin: 'elasticsearch.rest-test' dependencies { testCompile "org.elasticsearch.plugin:x-pack-core:${version}" + testCompile project(':client:rest-high-level') } String outputDir = "${buildDir}/generated-resources/${project.name}" diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java index 6a49e18ca93ef..5662990580f0f 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java +++ b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java @@ -7,35 +7,55 @@ import io.netty.util.ThreadDeathWatcher; import io.netty.util.concurrent.GlobalEventExecutor; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.common.network.NetworkAddress; -import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; +import org.elasticsearch.client.xpack.XPackUsageRequest; +import org.elasticsearch.client.xpack.XPackUsageResponse; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder; -import org.elasticsearch.xpack.core.action.XPackUsageResponse; -import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage; -import org.elasticsearch.xpack.core.security.SecurityField; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.yaml.ObjectPath; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.rules.ExternalResource; -import java.net.InetSocketAddress; -import java.util.Collection; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; +import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; /** * This test checks that a Monitoring's HTTP exporter correctly exports to a monitoring cluster @@ -46,7 +66,13 @@ * then uses a transport client to check that the data have been correctly received and * indexed in the cluster. */ -public class SmokeTestMonitoringWithSecurityIT extends ESIntegTestCase { +public class SmokeTestMonitoringWithSecurityIT extends ESRestTestCase { + + public class TestRestHighLevelClient extends RestHighLevelClient { + TestRestHighLevelClient() { + super(client(), RestClient::close, Collections.emptyList()); + } + } /** * A JUnit class level rule that runs after the AfterClass method in {@link ESIntegTestCase}, @@ -78,19 +104,45 @@ protected void after() { }; private static final String USER = "test_user"; - private static final String PASS = "x-pack-test-password"; + private static final SecureString PASS = new SecureString("x-pack-test-password".toCharArray()); + private static final String KEYSTORE_PASS = "testnode"; private static final String MONITORING_PATTERN = ".monitoring-*"; + static Path keyStore; + + @BeforeClass + public static void getKeyStore() { + try { + keyStore = PathUtils.get(SmokeTestMonitoringWithSecurityIT.class.getResource("/testnode.jks").toURI()); + } catch (URISyntaxException e) { + throw new ElasticsearchException("exception while reading the store", e); + } + if (!Files.exists(keyStore)) { + throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist."); + } + } + + @AfterClass + public static void clearKeyStore() { + keyStore = null; + } + + RestHighLevelClient newHighLevelClient() { + return new TestRestHighLevelClient(); + } + @Override - protected Collection> transportClientPlugins() { - return Collections.singletonList(XPackPlugin.class); + protected String getProtocol() { + return "https"; } @Override - protected Settings externalClusterClientSettings() { + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USER, PASS); return Settings.builder() - .put(SecurityField.USER_SETTING.getKey(), USER + ":" + PASS) - .put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4).build(); + .put(ThreadContext.PREFIX + ".Authorization", token) + .put(ESRestTestCase.TRUSTSTORE_PATH, keyStore) + .put(ESRestTestCase.TRUSTSTORE_PASSWORD, KEYSTORE_PASS).build(); } @Before @@ -100,73 +152,92 @@ public void enableExporter() throws Exception { .put("xpack.monitoring.exporters._http.enabled", true) .put("xpack.monitoring.exporters._http.host", "https://" + randomNodeHttpAddress()) .build(); - assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings)); + ClusterUpdateSettingsResponse response = newHighLevelClient().cluster().putSettings( + new ClusterUpdateSettingsRequest().transientSettings(exporterSettings), RequestOptions.DEFAULT); + assertTrue(response.isAcknowledged()); } @After - public void disableExporter() { + public void disableExporter() throws IOException { Settings exporterSettings = Settings.builder() .putNull("xpack.monitoring.collection.enabled") .putNull("xpack.monitoring.exporters._http.enabled") .putNull("xpack.monitoring.exporters._http.host") .build(); - assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings)); + ClusterUpdateSettingsResponse response = newHighLevelClient().cluster().putSettings( + new ClusterUpdateSettingsRequest().transientSettings(exporterSettings), RequestOptions.DEFAULT); + assertTrue(response.isAcknowledged()); } private boolean getMonitoringUsageExportersDefined() throws Exception { - final XPackUsageResponse usageResponse = new XPackUsageRequestBuilder(client()).execute().get(); - final Optional monitoringUsage = - usageResponse.getUsages() - .stream() - .filter(usage -> usage instanceof MonitoringFeatureSetUsage) - .map(usage -> (MonitoringFeatureSetUsage)usage) - .findFirst(); - - assertThat("Monitoring feature set does not exist", monitoringUsage.isPresent(), is(true)); - - return monitoringUsage.get().getExporters().isEmpty() == false; + RestHighLevelClient client = newHighLevelClient(); + final XPackUsageResponse usageResponse = client.xpack().usage(new XPackUsageRequest(), RequestOptions.DEFAULT); + Map monitoringUsage = usageResponse.getUsages().get("monitoring"); + assertThat("Monitoring feature set does not exist", monitoringUsage, notNullValue()); + + @SuppressWarnings("unchecked") + Map exporters = (Map) monitoringUsage.get("enabled_exporters"); + return exporters != null && exporters.isEmpty() == false; } public void testHTTPExporterWithSSL() throws Exception { // Ensures that the exporter is actually on assertBusy(() -> assertThat("[_http] exporter is not defined", getMonitoringUsageExportersDefined(), is(true))); + RestHighLevelClient client = newHighLevelClient(); // Checks that the monitoring index templates have been installed + GetIndexTemplatesRequest templateRequest = new GetIndexTemplatesRequest(MONITORING_PATTERN); assertBusy(() -> { - GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(MONITORING_PATTERN).get(); - assertThat(response.getIndexTemplates().size(), greaterThanOrEqualTo(2)); + try { + GetIndexTemplatesResponse response = client.indices().getIndexTemplate(templateRequest, RequestOptions.DEFAULT); + assertThat(response.getIndexTemplates().size(), greaterThanOrEqualTo(2)); + } catch (Exception e) { + fail("template not ready yet: " + e.getMessage()); + } }); + GetIndexRequest indexRequest = new GetIndexRequest(MONITORING_PATTERN); // Waits for monitoring indices to be created assertBusy(() -> { try { - assertThat(client().admin().indices().prepareExists(MONITORING_PATTERN).get().isExists(), equalTo(true)); + assertThat(client.indices().exists(indexRequest, RequestOptions.DEFAULT), equalTo(true)); } catch (Exception e) { - fail("exception when checking for monitoring documents: " + e.getMessage()); + fail("monitoring index not created yet: " + e.getMessage()); } }); // Waits for indices to be ready - ensureYellowAndNoInitializingShards(MONITORING_PATTERN); + ClusterHealthRequest healthRequest = new ClusterHealthRequest(MONITORING_PATTERN); + healthRequest.waitForStatus(ClusterHealthStatus.YELLOW); + healthRequest.waitForEvents(Priority.LANGUID); + healthRequest.waitForNoRelocatingShards(true); + healthRequest.waitForNoInitializingShards(true); + ClusterHealthResponse response = client.cluster().health(healthRequest, RequestOptions.DEFAULT); + assertThat(response.isTimedOut(), is(false)); // Checks that the HTTP exporter has successfully exported some data + SearchRequest searchRequest = new SearchRequest(new String[] { MONITORING_PATTERN }, new SearchSourceBuilder().size(0)); assertBusy(() -> { try { - assertThat(client().prepareSearch(MONITORING_PATTERN).setSize(0).get().getHits().getTotalHits().value, greaterThan(0L)); + assertThat(client.search(searchRequest, RequestOptions.DEFAULT).getHits().getTotalHits().value, greaterThan(0L)); } catch (Exception e) { - fail("exception when checking for monitoring documents: " + e.getMessage()); + fail("monitoring date not exported yet: " + e.getMessage()); } }); } - private String randomNodeHttpAddress() { - List nodes = client().admin().cluster().prepareNodesInfo().clear().setHttp(true).get().getNodes(); - assertThat(nodes.size(), greaterThan(0)); - - InetSocketAddress[] httpAddresses = new InetSocketAddress[nodes.size()]; - for (int i = 0; i < nodes.size(); i++) { - httpAddresses[i] = nodes.get(i).getHttp().address().publishAddress().address(); + private String randomNodeHttpAddress() throws IOException { + Response response = client().performRequest(new Request("GET", "/_nodes")); + assertOK(response); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + Map nodesAsMap = objectPath.evaluate("nodes"); + List httpAddresses = new ArrayList<>(); + for (Map.Entry entry : nodesAsMap.entrySet()) { + Map nodeDetails = (Map) entry.getValue(); + Map httpInfo = (Map) nodeDetails.get("http"); + httpAddresses.add((String) httpInfo.get("publish_address")); } - return NetworkAddress.format(randomFrom(httpAddresses)); + assertThat(httpAddresses.size(), greaterThan(0)); + return randomFrom(httpAddresses); } } diff --git a/x-pack/qa/transport-client-tests/build.gradle b/x-pack/qa/transport-client-tests/build.gradle deleted file mode 100644 index 5ca96eb0d7a87..0000000000000 --- a/x-pack/qa/transport-client-tests/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' - -dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" - testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') -} - -integTestCluster { - setting 'xpack.security.enabled', 'false' - setting 'xpack.license.self_generated.type', 'trial' -} - - -testingConventions { - naming.clear() - naming { - IT { - baseClass 'org.elasticsearch.xpack.ml.client.ESXPackSmokeClientTestCase' - } - } -} \ No newline at end of file diff --git a/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java b/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java deleted file mode 100644 index 28267614dd36d..0000000000000 --- a/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.client; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.env.Environment; -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.URL; -import java.nio.file.Path; -import java.util.Locale; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; -import static org.hamcrest.Matchers.notNullValue; - -/** - * An abstract base class to run integration tests against an Elasticsearch - * cluster running outside of the test process. - *

- * You can define a list of transport addresses from where you can reach your - * cluster by setting "tests.cluster" system property. It defaults to - * "localhost:9300". If you run this from `gradle integTest` then it will start - * the clsuter for you and set up the property. - *

- * If you want to debug this module from your IDE, then start an external - * cluster by yourself, maybe with `gradle run`, then run JUnit. If you changed - * the default port, set "-Dtests.cluster=localhost:PORT" when running your - * test. - */ -@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") -public abstract class ESXPackSmokeClientTestCase extends LuceneTestCase { - - /** - * Key used to eventually switch to using an external cluster and provide - * its transport addresses - */ - public static final String TESTS_CLUSTER = "tests.cluster"; - - protected static final Logger logger = LogManager.getLogger(ESXPackSmokeClientTestCase.class); - - private static final AtomicInteger counter = new AtomicInteger(); - private static Client client; - private static String clusterAddresses; - protected String index; - - private static Client startClient(Path tempDir, TransportAddress... transportAddresses) { - Settings.Builder builder = Settings.builder() - .put("node.name", "qa_xpack_smoke_client_" + counter.getAndIncrement()) - .put("client.transport.ignore_cluster_name", true) - .put("xpack.security.enabled", false) - .put(Environment.PATH_HOME_SETTING.getKey(), tempDir); - TransportClient client = new PreBuiltXPackTransportClient(builder.build()) - .addTransportAddresses(transportAddresses); - - logger.info("--> Elasticsearch Java TransportClient started"); - - Exception clientException = null; - try { - ClusterHealthResponse health = client.admin().cluster().prepareHealth().get(); - logger.info("--> connected to [{}] cluster which is running [{}] node(s).", - health.getClusterName(), health.getNumberOfNodes()); - } catch (Exception e) { - logger.error("Error getting cluster health", e); - clientException = e; - } - - assumeNoException("Sounds like your cluster is not running at " + clusterAddresses, - clientException); - - return client; - } - - private static Client startClient() throws IOException { - String[] stringAddresses = clusterAddresses.split(","); - TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; - int i = 0; - for (String stringAddress : stringAddresses) { - URL url = new URL("http://" + stringAddress); - InetAddress inetAddress = InetAddress.getByName(url.getHost()); - transportAddresses[i++] = new TransportAddress( - new InetSocketAddress(inetAddress, url.getPort())); - } - return startClient(createTempDir(), transportAddresses); - } - - public static Client getClient() { - if (client == null) { - try { - client = startClient(); - } catch (IOException e) { - logger.error("can not start the client", e); - } - assertThat(client, notNullValue()); - } - return client; - } - - @BeforeClass - public static void initializeSettings() { - clusterAddresses = System.getProperty(TESTS_CLUSTER); - if (clusterAddresses == null || clusterAddresses.isEmpty()) { - fail("Must specify " + TESTS_CLUSTER + " for smoke client test"); - } - } - - @AfterClass - public static void stopTransportClient() { - if (client != null) { - client.close(); - client = null; - } - } - - @Before - public void defineIndexName() { - doClean(); - index = "qa-xpack-smoke-test-client-" - + randomAsciiOfLength(10).toLowerCase(Locale.getDefault()); - } - - @After - public void cleanIndex() { - doClean(); - } - - private void doClean() { - if (client != null) { - try { - client.admin().indices().prepareDelete(index).get(); - } catch (Exception e) { - // We ignore this cleanup exception - } - } - } -} diff --git a/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java b/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java deleted file mode 100644 index 1a4959c0be84a..0000000000000 --- a/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.client; - -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.xpack.core.XPackClient; -import org.elasticsearch.xpack.core.ml.action.CloseJobAction; -import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; -import org.elasticsearch.xpack.core.ml.action.FlushJobAction; -import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; -import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsAction; -import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction; -import org.elasticsearch.xpack.core.ml.action.OpenJobAction; -import org.elasticsearch.xpack.core.ml.action.PostDataAction; -import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.PutJobAction; -import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction; -import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction; -import org.elasticsearch.xpack.core.ml.client.MachineLearningClient; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.DataDescription; -import org.elasticsearch.xpack.core.ml.job.config.Detector; -import org.elasticsearch.xpack.core.ml.job.config.Job; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; - -public class MLTransportClientIT extends ESXPackSmokeClientTestCase { - - public void testMLTransportClient_JobActions() { - Client client = getClient(); - XPackClient xPackClient = new XPackClient(client); - MachineLearningClient mlClient = xPackClient.machineLearning(); - - String jobId = "ml-transport-client-it-job"; - Job.Builder job = createJob(jobId); - - PutJobAction.Response putJobResponse = mlClient.putJob(new PutJobAction.Request(job)).actionGet(); - assertThat(putJobResponse, notNullValue()); - - GetJobsAction.Response getJobResponse = mlClient.getJobs(new GetJobsAction.Request(jobId)).actionGet(); - assertThat(getJobResponse, notNullValue()); - assertThat(getJobResponse.getResponse(), notNullValue()); - assertThat(getJobResponse.getResponse().count(), equalTo(1L)); - - // Open job POST data, flush, close and check a result - AcknowledgedResponse openJobResponse = mlClient.openJob(new OpenJobAction.Request(jobId)).actionGet(); - assertThat(openJobResponse.isAcknowledged(), equalTo(true)); - - String content = "{\"time\":1000, \"msg\": \"some categorical message\"}\n" + - "{\"time\":11000, \"msg\": \"some categorical message in the second bucket\"}\n" + - "{\"time\":21000, \"msg\": \"some categorical message in the third bucket\"}\n"; - PostDataAction.Request postRequest = new PostDataAction.Request(jobId); - postRequest.setContent(new BytesArray(content), XContentType.JSON); - PostDataAction.Response postResponse = mlClient.postData(postRequest).actionGet(); - assertThat(postResponse.getDataCounts(), notNullValue()); - assertThat(postResponse.getDataCounts().getInputFieldCount(), equalTo(3L)); - - FlushJobAction.Response flushResponse = mlClient.flushJob(new FlushJobAction.Request(jobId)).actionGet(); - assertThat(flushResponse.isFlushed(), equalTo(true)); - - CloseJobAction.Response closeResponse = mlClient.closeJob(new CloseJobAction.Request(jobId)).actionGet(); - assertThat(closeResponse.isClosed(), equalTo(true)); - - GetBucketsAction.Response getBucketsResponse = mlClient.getBuckets(new GetBucketsAction.Request(jobId)).actionGet(); - assertThat(getBucketsResponse.getBuckets().count(), equalTo(1L)); - - // Update a model snapshot - GetModelSnapshotsAction.Response getModelSnapshotResponse = - mlClient.getModelSnapshots(new GetModelSnapshotsAction.Request(jobId, null)).actionGet(); - assertThat(getModelSnapshotResponse.getPage().count(), equalTo(1L)); - String snapshotId = getModelSnapshotResponse.getPage().results().get(0).getSnapshotId(); - - UpdateModelSnapshotAction.Request updateModelSnapshotRequest = new UpdateModelSnapshotAction.Request(jobId, snapshotId); - updateModelSnapshotRequest.setDescription("Changed description"); - UpdateModelSnapshotAction.Response updateModelSnapshotResponse = - mlClient.updateModelSnapshot(updateModelSnapshotRequest).actionGet(); - assertThat(updateModelSnapshotResponse.getModel(), notNullValue()); - assertThat(updateModelSnapshotResponse.getModel().getDescription(), equalTo("Changed description")); - - // and delete the job - AcknowledgedResponse deleteJobResponse = mlClient.deleteJob(new DeleteJobAction.Request(jobId)).actionGet(); - assertThat(deleteJobResponse, notNullValue()); - assertThat(deleteJobResponse.isAcknowledged(), equalTo(true)); - } - - public void testMLTransportClient_ValidateActions() { - Client client = getClient(); - XPackClient xPackClient = new XPackClient(client); - MachineLearningClient mlClient = xPackClient.machineLearning(); - - Detector.Builder detector = new Detector.Builder(); - detector.setFunction("count"); - ValidateDetectorAction.Request validateDetectorRequest = new ValidateDetectorAction.Request(detector.build()); - AcknowledgedResponse validateDetectorResponse = mlClient.validateDetector(validateDetectorRequest).actionGet(); - assertThat(validateDetectorResponse.isAcknowledged(), equalTo(true)); - - Job.Builder job = createJob("ml-transport-client-it-validate-job"); - ValidateJobConfigAction.Request validateJobRequest = new ValidateJobConfigAction.Request(job.build(new Date())); - AcknowledgedResponse validateJobResponse = mlClient.validateJobConfig(validateJobRequest).actionGet(); - assertThat(validateJobResponse.isAcknowledged(), equalTo(true)); - } - - - public void testMLTransportClient_DateFeedActions() { - Client client = getClient(); - XPackClient xPackClient = new XPackClient(client); - MachineLearningClient mlClient = xPackClient.machineLearning(); - - String jobId = "ml-transport-client-it-datafeed-job"; - Job.Builder job = createJob(jobId); - - PutJobAction.Response putJobResponse = mlClient.putJob(new PutJobAction.Request(job)).actionGet(); - assertThat(putJobResponse, notNullValue()); - - String datafeedId = "ml-transport-client-it-datafeed"; - DatafeedConfig.Builder datafeed = new DatafeedConfig.Builder(datafeedId, jobId); - String datafeedIndex = "ml-transport-client-test"; - String datatype = "type-bar"; - datafeed.setIndices(Collections.singletonList(datafeedIndex)); - - mlClient.putDatafeed(new PutDatafeedAction.Request(datafeed.build())).actionGet(); - - GetDatafeedsAction.Response getDatafeedResponse = mlClient.getDatafeeds(new GetDatafeedsAction.Request(datafeedId)).actionGet(); - assertThat(getDatafeedResponse.getResponse(), notNullValue()); - - // Open job before starting the datafeed - AcknowledgedResponse openJobResponse = mlClient.openJob(new OpenJobAction.Request(jobId)).actionGet(); - assertThat(openJobResponse.isAcknowledged(), equalTo(true)); - - // create the index for the data feed - Map source = new HashMap<>(); - source.put("time", new Date()); - source.put("message", "some message"); - client.prepareIndex(datafeedIndex, datatype).setSource(source).get(); - - StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(datafeedId, new Date().getTime()); - AcknowledgedResponse startDataFeedResponse = mlClient.startDatafeed(startDatafeedRequest).actionGet(); - assertThat(startDataFeedResponse.isAcknowledged(), equalTo(true)); - - StopDatafeedAction.Response stopDataFeedResponse = mlClient.stopDatafeed(new StopDatafeedAction.Request(datafeedId)).actionGet(); - assertThat(stopDataFeedResponse.isStopped(), equalTo(true)); - } - - private Job.Builder createJob(String jobId) { - Job.Builder job = new Job.Builder(); - job.setId(jobId); - - List detectors = new ArrayList<>(); - Detector.Builder detector = new Detector.Builder(); - detector.setFunction("count"); - detectors.add(detector.build()); - - AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(detectors); - analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(10L)); - job.setAnalysisConfig(analysisConfig); - job.setDataDescription(new DataDescription.Builder()); - return job; - } -} diff --git a/x-pack/transport-client/build.gradle b/x-pack/transport-client/build.gradle deleted file mode 100644 index d764ef897447a..0000000000000 --- a/x-pack/transport-client/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -apply plugin: 'elasticsearch.build' -apply plugin: 'nebula.maven-base-publish' -apply plugin: 'nebula.maven-scm' - -group = 'org.elasticsearch.client' -archivesBaseName = 'x-pack-transport' - -dependencies { - // this "api" dependency looks weird, but it is correct, as it contains - // all of x-pack for now, and transport client will be going away in the future. - compile "org.elasticsearch.plugin:x-pack-core:${version}" - compile "org.elasticsearch.client:transport:${version}" - testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" - testCompile "junit:junit:${versions.junit}" - testCompile "org.hamcrest:hamcrest:${versions.hamcrest}" -} - -dependencyLicenses.enabled = false - -forbiddenApisTest { - // we don't use the core test-framework, no lucene classes present so we don't want the es-test-signatures to - // be pulled in - replaceSignatureFiles 'jdk-signatures', 'es-all-signatures' -} - -testingConventions { - naming.clear() - naming { - Tests { - baseClass 'com.carrotsearch.randomizedtesting.RandomizedTest' - } - } -} - -publishing { - publications { - nebula(MavenPublication) { - artifactId = archivesBaseName - } - } -} diff --git a/x-pack/transport-client/src/main/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClient.java b/x-pack/transport-client/src/main/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClient.java deleted file mode 100644 index cf4e5db92b00e..0000000000000 --- a/x-pack/transport-client/src/main/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClient.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.client; - -import io.netty.util.ThreadDeathWatcher; -import io.netty.util.concurrent.GlobalEventExecutor; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.client.PreBuiltTransportClient; -import org.elasticsearch.xpack.core.XPackClientPlugin; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.security.SecurityField; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.concurrent.TimeUnit; - -/** - * A builder to create an instance of {@link TransportClient} that pre-installs - * all of the plugins installed by the {@link PreBuiltTransportClient} and the - * {@link XPackPlugin} so that the client may be used with an x-pack enabled - * cluster. - * - * @deprecated {@link TransportClient} is deprecated in favour of the high-level REST client and will be removed in Elasticsearch 8.0 - */ -@SuppressWarnings({"unchecked","varargs"}) -@Deprecated -public class PreBuiltXPackTransportClient extends PreBuiltTransportClient { - - @SafeVarargs - public PreBuiltXPackTransportClient(Settings settings, Class... plugins) { - this(settings, Arrays.asList(plugins)); - } - - public PreBuiltXPackTransportClient(Settings settings, Collection> plugins) { - this(settings, plugins, null); - } - - public PreBuiltXPackTransportClient(Settings settings, Collection> plugins, - HostFailureListener hostFailureListener) { - super(settings, addPlugins(plugins, Collections.singletonList(XPackClientPlugin.class)), hostFailureListener); - } - - @Override - public void close() { - super.close(); - if (NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(SecurityField.NAME4)) { - try { - GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } -} diff --git a/x-pack/transport-client/src/test/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClientTests.java b/x-pack/transport-client/src/test/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClientTests.java deleted file mode 100644 index f9808ce54faac..0000000000000 --- a/x-pack/transport-client/src/test/java/org/elasticsearch/xpack/client/PreBuiltXPackTransportClientTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.client; - -import com.carrotsearch.randomizedtesting.RandomizedTest; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.core.security.SecurityField; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * Unit tests for the {@link PreBuiltXPackTransportClient} - */ -public class PreBuiltXPackTransportClientTests extends RandomizedTest { - - @Test - public void testPluginInstalled() { - try (TransportClient client = new PreBuiltXPackTransportClient(Settings.EMPTY)) { - Settings settings = client.settings(); - assertEquals(SecurityField.NAME4, NetworkModule.TRANSPORT_TYPE_SETTING.get(settings)); - } - } - -} \ No newline at end of file