Merge branch 'master' into zen2_detach_cluster

.ci/packer_cache.sh

@@ -16,4 +16,9 @@ while [ -h "$SCRIPT" ] ; do
 done
 
 source $(dirname "${SCRIPT}")/java-versions.properties
-JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} ./gradlew --parallel  resolveAllDependencies composePull
+export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
+# We are caching BWC versions too, need these so we can build those
+export JAVA8_HOME="${HOME}"/.java/java8
+export JAVA11_HOME="${HOME}"/.java/java11
+export JAVA12_HOME="${HOME}"/.java/java12
+./gradlew --parallel  clean pullFixture  --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies

buildSrc/build.gradle

@@ -183,8 +183,12 @@ if (project != rootProject) {
   }
 
   dependencies {
-    distribution project(':distribution:archives:zip')
-    distribution project(':distribution:archives:oss-zip')
+    distribution project(':distribution:archives:windows-zip')
+    distribution project(':distribution:archives:oss-windows-zip')
+    distribution project(':distribution:archives:darwin-tar')
+    distribution project(':distribution:archives:oss-darwin-tar')
+    distribution project(':distribution:archives:linux-tar')
+    distribution project(':distribution:archives:oss-linux-tar')
   }
 
   String localDownloads = "${rootProject.buildDir}/local-downloads"

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -191,25 +191,38 @@ class ClusterFormationTasks {
             throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
         }
         Version version = Version.fromString(elasticsearchVersion)
-        String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
+        String os = getOs()
+        String classifier = "${os}-x86_64"
+        String packaging = os.equals('windows') ? 'zip' : 'tar.gz'
         String artifactName = 'elasticsearch'
         if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
         Object dependency
+        String snapshotProject = "${os}-${os.equals('windows') ? 'zip' : 'tar'}"
+        if (version.before("7.0.0")) {
+            snapshotProject = "zip"
+        }
+        if (distro.equals("oss")) {
+            snapshotProject = "oss-" + snapshotProject
+        }
         boolean internalBuild = project.hasProperty('bwcVersions')
         VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
         if (project.hasProperty('bwcVersions')) {
             // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
             unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
         }
         if (unreleasedInfo != null) {
-            dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
+            dependency = project.dependencies.project(
+                        path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
         } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
             dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
         } else {
-            dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
+            if (version.before('7.0.0')) {
+                classifier = "" // for bwc, before we had classifiers
+            }
+            // group does not matter as it is not used when we pull from the ivy repo that points to the download service
+            dependency = "dnm:${artifactName}:${elasticsearchVersion}${classifier}@${packaging}"
         }
         project.dependencies.add(configuration.name, dependency)
     }
@@ -335,8 +348,15 @@ class ClusterFormationTasks {
           the elasticsearch source tree then this should be the version of elasticsearch built by the source tree.
           If it isn't then Bad Things(TM) will happen. */
         Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-            from {
-                project.zipTree(configuration.singleFile)
+            if (getOs().equals("windows")) {
+                from {
+                    project.zipTree(configuration.singleFile)
+                }
+            } else {
+                // macos and linux use tar
+                from {
+                    project.tarTree(project.resources.gzip(configuration.singleFile))
+                }
             }
             into node.baseDir
         }
@@ -948,4 +968,15 @@ class ClusterFormationTasks {
         PluginPropertiesExtension extension = pluginProject.extensions.findByName('esplugin')
         return extension.name
     }
+
+    /** Find the current OS */
+    static String getOs() {
+        String os = "linux"
+        if (Os.FAMILY_WINDOWS) {
+            os = "windows"
+        } else if (Os.FAMILY_MAC) {
+            os = "darwin"
+        }
+        return os
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -53,10 +53,10 @@ class VagrantTestPlugin implements Plugin<Project> {
 
     /** All distributions to bring into test VM, whether or not they are used **/
     static final List<String> DISTRIBUTIONS = unmodifiableList([
-            'archives:tar',
-            'archives:oss-tar',
-            'archives:zip',
-            'archives:oss-zip',
+            'archives:linux-tar',
+            'archives:oss-linux-tar',
+            'archives:windows-zip',
+            'archives:oss-windows-zip',
             'packages:rpm',
             'packages:oss-rpm',
             'packages:deb',

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -59,14 +59,17 @@ public void apply(Project project) {
             disableTaskByType(tasks, JarHellTask.class);
 
             Task buildFixture = project.getTasks().create("buildFixture");
+            Task pullFixture = project.getTasks().create("pullFixture");
             Task preProcessFixture = project.getTasks().create("preProcessFixture");
             buildFixture.dependsOn(preProcessFixture);
+            pullFixture.dependsOn(preProcessFixture);
             Task postProcessFixture = project.getTasks().create("postProcessFixture");
 
             if (dockerComposeSupported(project) == false) {
                 preProcessFixture.setEnabled(false);
                 postProcessFixture.setEnabled(false);
                 buildFixture.setEnabled(false);
+                pullFixture.setEnabled(false);
                 return;
             }
 
@@ -81,7 +84,9 @@ public void apply(Project project) {
             );
 
             buildFixture.dependsOn(tasks.getByName("composeUp"));
+            pullFixture.dependsOn(tasks.getByName("composePull"));
             tasks.getByName("composeUp").mustRunAfter(preProcessFixture);
+            tasks.getByName("composePull").mustRunAfter(preProcessFixture);
             postProcessFixture.dependsOn(buildFixture);
 
             configureServiceInfoForTask(

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java

@@ -46,6 +46,8 @@ static Request putFollow(PutFollowRequest putFollowRequest) throws IOException {
             .addPathPartAsIs("_ccr", "follow")
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        parameters.withWaitForActiveShards(putFollowRequest.waitForActiveShards());
         request.setEntity(createEntity(putFollowRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -49,8 +49,6 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
-import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -61,7 +59,9 @@
 import org.elasticsearch.client.indices.GetIndexTemplatesRequest;
 import org.elasticsearch.client.indices.GetMappingsRequest;
 import org.elasticsearch.client.indices.GetMappingsResponse;
+import org.elasticsearch.client.indices.GetIndexTemplatesResponse;
 import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
+import org.elasticsearch.client.indices.PutIndexTemplateRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
 import org.elasticsearch.client.indices.UnfreezeIndexRequest;
 import org.elasticsearch.rest.RestStatus;
@@ -908,6 +908,7 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques
             AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
+
     /**
      * Puts an index template using the Index Templates API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
@@ -916,9 +917,13 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
+     * @deprecated This old form of request allows types in mappings. Use {@link #putTemplate(PutIndexTemplateRequest, RequestOptions)} 
+     * instead which introduces a new request object without types.
      */
-    public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest,
-                                            RequestOptions options) throws IOException {
+    @Deprecated
+    public AcknowledgedResponse putTemplate(
+            org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest,
+            RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options,
             AcknowledgedResponse::fromXContent, emptySet());
     }
@@ -930,9 +935,44 @@ public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplate
      * @param putIndexTemplateRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
+     * @deprecated This old form of request allows types in mappings. 
+     * Use {@link #putTemplateAsync(PutIndexTemplateRequest, RequestOptions, ActionListener)} 
+     * instead which introduces a new request object without types.
      */
-    public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options,
-                                 ActionListener<AcknowledgedResponse> listener) {
+    @Deprecated
+    public void putTemplateAsync(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, 
+            RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options,
+            AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
+
+    /**
+     * Puts an index template using the Index Templates API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
+     * on elastic.co</a>
+     * @param putIndexTemplateRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse putTemplate(
+            PutIndexTemplateRequest putIndexTemplateRequest,
+            RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options,
+            AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously puts an index template using the Index Templates API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
+     * on elastic.co</a>
+     * @param putIndexTemplateRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, 
+            RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options,
             AcknowledgedResponse::fromXContent, listener, emptySet());
     }
@@ -968,33 +1008,74 @@ public void validateQueryAsync(ValidateQueryRequest validateQueryRequest, Reques
     }
 
     /**
-     * Gets index templates using the Index Templates API
+     * Gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, where the
+     * mapping definition is nested under the type name.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
      * on elastic.co</a>
      * @param getIndexTemplatesRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
+     * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use 
+     * {@link #getIndexTemplate(GetIndexTemplatesRequest, RequestOptions)} instead which returns a new response object
      */
-    public GetIndexTemplatesResponse getTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest,
-                                                 RequestOptions options) throws IOException {
-        return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates,
-            options, GetIndexTemplatesResponse::fromXContent, emptySet());
+    @Deprecated
+    public org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate(
+            GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest,
+            IndicesRequestConverters::getTemplatesWithDocumentTypes,
+            options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, emptySet());
     }
+
+    /**
+     * Gets index templates using the Index Templates API
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
+     * on elastic.co</a>
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param getIndexTemplatesRequest the request
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetIndexTemplatesResponse getIndexTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options)
+                                                  throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest,
+            IndicesRequestConverters::getTemplates,
+            options, GetIndexTemplatesResponse::fromXContent, emptySet());
+    }    
 
     /**
-     * Asynchronously gets index templates using the Index Templates API
+     * Asynchronously gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, 
+     * where the mapping definition is nested under the type name.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
      * on elastic.co</a>
      * @param getIndexTemplatesRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
+     * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use 
+     * {@link #getIndexTemplateAsync(GetIndexTemplatesRequest, RequestOptions, ActionListener)} instead which returns a new response object
      */
+    @Deprecated
     public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options,
+                                 ActionListener<org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest,
+            IndicesRequestConverters::getTemplatesWithDocumentTypes,
+            options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Asynchronously gets index templates using the Index Templates API
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
+     * on elastic.co</a>
+     * @param getIndexTemplatesRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getIndexTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, 
                                  ActionListener<GetIndexTemplatesResponse> listener) {
-        restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates,
+        restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest,
+            IndicesRequestConverters::getTemplates,
             options, GetIndexTemplatesResponse::fromXContent, listener, emptySet());
-    }
+    }    
 
     /**
      * Uses the Index Templates API to determine if index templates exist