Merge branch 'master' into system-jvm-options

.ci/os.ps1

@@ -27,10 +27,6 @@ New-Item -ItemType directory -Path \tmp
 
 $ErrorActionPreference="Continue"
 # TODO: remove the task exclusions once dependencies are set correctly and these don't run for Windows or buldiung the deb on windows is fixed
-& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest `
-   -x :distribution:packages:buildOssDeb `
-   -x :distribution:packages:buildDeb `
-   -x :distribution:packages:buildOssRpm `
-   -x :distribution:packages:buildRpm `
+& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest
 
 exit $LastExitCode

TESTING.asciidoc

@@ -43,6 +43,12 @@ supports a remote debugging option:
 ./gradlew run --debug-jvm
 ---------------------------------------------------------------------------
 
+This will instruct all JVMs (including any that run cli tools such as creating the keyring or adding users)
+to suspend and initiate a debug connection on port incrementing from `5005`.
+As such the IDE needs to be instructed to listen for connections on this port.
+Since we might run multiple JVMs as part of configuring and starting the cluster it's
+recommended to have the option to aut restart checked.
+
 ==== Distribution
 
 By default a node is started with the zip distribution.

build.gradle

@@ -32,7 +32,7 @@ import org.gradle.util.GradleVersion
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 
 plugins {
-    id 'com.gradle.build-scan' version '2.4'
+    id 'com.gradle.build-scan' version '2.4.2'
     id 'lifecycle-base'
     id 'elasticsearch.global-build-info'
     id "com.diffplug.gradle.spotless" version "3.24.2" apply false

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -36,7 +36,7 @@ public Boolean getDebug() {
 
     @Override
     public void beforeStart() {
-        int debugPort = 8000;
+        int debugPort = 5005;
         int httpPort = 9200;
         int transportPort = 9300;
         Map<String, String> additionalSettings = System.getProperties().entrySet().stream()
@@ -57,7 +57,7 @@ public void beforeStart() {
                         "Running elasticsearch in debug mode, {} suspending until connected on debugPort {}",
                         node, debugPort
                     );
-                    node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=" + debugPort);
+                    node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=" + debugPort);
                     debugPort += 1;
                 }
             }

client/rest-high-level/build.gradle

@@ -129,8 +129,11 @@ testClusters.integTest {
 
   setting 'indices.lifecycle.poll_interval', '1000ms'
   keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
+  extraConfigFile 'roles.yml', file('roles.yml')
   user username:  System.getProperty('tests.rest.cluster.username', 'test_user'),
-          password:  System.getProperty('tests.rest.cluster.password', 'test-password')
+          password:  System.getProperty('tests.rest.cluster.password', 'test-password'),
+          role:  System.getProperty('tests.rest.cluster.role', 'admin')
+  user username:  'admin_user', password: 'admin-password'
 
   extraConfigFile nodeCert.name, nodeCert
   extraConfigFile nodeTrustStore.name, nodeTrustStore

client/rest-high-level/roles.yml

@@ -0,0 +1,12 @@
+admin:
+  cluster:
+    - all
+  indices:
+    - names: '*'
+      privileges:
+        - all
+  run_as: [ '*' ]
+  applications:
+    - application: '*'
+      privileges: [ '*' ]
+      resources: [ '*' ]

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -297,55 +297,6 @@ public Cancellable getMappingAsync(GetMappingsRequest getMappingsRequest, Reques
             emptySet());
     }
 
-    /**
-     * Retrieves the mappings on an index or indices using the Get Mapping API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-mapping.html">
-     * Get Mapping API on elastic.co</a>
-     * @param getMappingsRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     * @throws IOException in case there is a problem sending the request or parsing back the response
-     *
-     * @deprecated This method uses old request and response objects which still refer to types, a deprecated
-     * feature. The method {@link #getMapping(GetMappingsRequest, RequestOptions)} should be used instead, which
-     * accepts a new request object.
-     */
-    @Deprecated
-    public org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse getMapping(
-            org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest getMappingsRequest,
-            RequestOptions options) throws IOException {
-        return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest,
-            IndicesRequestConverters::getMappings,
-            options,
-            org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse::fromXContent,
-            emptySet());
-    }
-
-    /**
-     * Asynchronously retrieves the mappings on an index on indices using the Get Mapping API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-mapping.html">
-     * Get Mapping API on elastic.co</a>
-     * @param getMappingsRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     *
-     * @deprecated This method uses old request and response objects which still refer to types, a deprecated feature.
-     * The method {@link #getMapping(GetMappingsRequest, RequestOptions)} should be used instead, which accepts a new
-     * request object.
-     * @return cancellable that may be used to cancel the request
-     */
-    @Deprecated
-    public Cancellable getMappingAsync(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest getMappingsRequest,
-                                       RequestOptions options,
-                                       ActionListener<org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse> listener) {
-        return restHighLevelClient.performRequestAsyncAndParseEntity(getMappingsRequest,
-            IndicesRequestConverters::getMappings,
-            options,
-            org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse::fromXContent,
-            listener,
-            emptySet());
-    }
-
     /**
      * Retrieves the field mappings on an index or indices using the Get Field Mapping API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-field-mapping.html">
@@ -679,42 +630,6 @@ public Cancellable getAsync(GetIndexRequest getIndexRequest, RequestOptions opti
             GetIndexResponse::fromXContent, listener, emptySet());
     }
 
-    /**
-     * Retrieve information about one or more indexes
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-index.html">
-     * Indices Get Index API on elastic.co</a>
-     * @param getIndexRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     * @throws IOException in case there is a problem sending the request or parsing back the response
-     * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method
-     * {@link #get(GetIndexRequest, RequestOptions)} should be used instead, which accepts a new request object.
-     */
-    @Deprecated
-    public org.elasticsearch.action.admin.indices.get.GetIndexResponse get(
-            org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest, RequestOptions options) throws IOException {
-        return restHighLevelClient.performRequestAndParseEntity(getIndexRequest, IndicesRequestConverters::getIndex, options,
-                org.elasticsearch.action.admin.indices.get.GetIndexResponse::fromXContent, emptySet());
-    }
-
-    /**
-     * Retrieve information about one or more indexes
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-index.html">
-     * Indices Get Index API on elastic.co</a>
-     * @param getIndexRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method
-     * {@link #getAsync(GetIndexRequest, RequestOptions, ActionListener)} should be used instead, which accepts a new request object.
-     * @return cancellable that may be used to cancel the request
-     */
-    @Deprecated
-    public Cancellable getAsync(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest, RequestOptions options,
-                                ActionListener<org.elasticsearch.action.admin.indices.get.GetIndexResponse> listener) {
-        return restHighLevelClient.performRequestAsyncAndParseEntity(getIndexRequest, IndicesRequestConverters::getIndex, options,
-                org.elasticsearch.action.admin.indices.get.GetIndexResponse::fromXContent, listener, emptySet());
-    }
-
     /**
      * Force merge one or more indices using the Force Merge API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html">

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java

@@ -189,22 +189,6 @@ static Request getMappings(GetMappingsRequest getMappingsRequest) {
         return request;
     }
 
-    @Deprecated
-    static Request getMappings(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest getMappingsRequest) {
-        String[] indices = getMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.indices();
-        String[] types = getMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.types();
-
-        Request request = new Request(HttpGet.METHOD_NAME, RequestConverters.endpoint(indices, "_mapping", types));
-
-        RequestConverters.Params parameters = new RequestConverters.Params();
-        parameters.withMasterTimeout(getMappingsRequest.masterNodeTimeout());
-        parameters.withIndicesOptions(getMappingsRequest.indicesOptions());
-        parameters.withLocal(getMappingsRequest.local());
-        parameters.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
-        request.addParameters(parameters.asMap());
-        return request;
-    }
-
     static Request getFieldMapping(GetFieldMappingsRequest getFieldMappingsRequest) {
         String[] indices = getFieldMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.indices();
         String[] fields = getFieldMappingsRequest.fields() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.fields();