Merge remote-tracking branch 'origin/master' into snapshot-lifecycle-…

…management

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -214,9 +214,15 @@ class VagrantTestPlugin implements Plugin<Project> {
             } else {
                 UPGRADE_FROM_ARCHIVES.each {
                     // The version of elasticsearch that we upgrade *from*
-                    dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
-                    if (upgradeFromVersion.onOrAfter('6.3.0')) {
-                        dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                    if (upgradeFromVersion.onOrAfter('7.0.0')) {
+                        String arch = it == "rpm" ? "x86_64" : "amd64"
+                        dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}-${arch}@${it}")
+                        dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}-${arch}@${it}")
+                    } else {
+                        dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                        if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                            dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                        }
                     }
                 }
             }

client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java

@@ -133,9 +133,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startArray(FAILURES.getPreferredName());
         if (shardFailures != null) {
             for (ShardOperationFailedException shardFailure : shardFailures) {
-                builder.startObject();
                 shardFailure.toXContent(builder, params);
-                builder.endObject();
             }
         }
         builder.endArray();

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java

@@ -31,7 +31,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -56,7 +56,7 @@
 /**
  * A request to create an index template.
  */
-public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateRequest> implements IndicesRequest, ToXContent {
+public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateRequest> implements IndicesRequest, ToXContentFragment {
 
     private String name;
 
@@ -191,7 +191,7 @@ public PutIndexTemplateRequest settings(Map<String, Object> source) {
     public Settings settings() {
         return this.settings;
     }
-    
+
     /**
      * Adds mapping that will be added when the index gets created.
      *
@@ -201,7 +201,7 @@ public Settings settings() {
     public PutIndexTemplateRequest mapping(String source, XContentType xContentType) {
         internalMapping(XContentHelper.convertToMap(new BytesArray(source), true, xContentType).v2());
         return this;
-    }    
+    }
 
     /**
      * The cause for this index template creation.
@@ -221,11 +221,11 @@ public String cause() {
      * @param source The mapping source
      */
     public PutIndexTemplateRequest mapping(XContentBuilder source) {
-        internalMapping(XContentHelper.convertToMap(BytesReference.bytes(source), 
+        internalMapping(XContentHelper.convertToMap(BytesReference.bytes(source),
                 true, source.contentType()).v2());
-        return this;        
-    }    
-    
+        return this;
+    }
+
     /**
      * Adds mapping that will be added when the index gets created.
      *
@@ -235,16 +235,16 @@ public PutIndexTemplateRequest mapping(XContentBuilder source) {
     public PutIndexTemplateRequest mapping(BytesReference source, XContentType xContentType) {
         internalMapping(XContentHelper.convertToMap(source, true, xContentType).v2());
         return this;
-    } 
-    
+    }
+
     /**
      * Adds mapping that will be added when the index gets created.
      *
      * @param source The mapping source
      */
     public PutIndexTemplateRequest mapping(Map<String, Object> source) {
         return internalMapping(source);
-    }      
+    }
 
     private PutIndexTemplateRequest internalMapping(Map<String, Object> source) {
         try {
@@ -257,12 +257,12 @@ private PutIndexTemplateRequest internalMapping(Map<String, Object> source) {
                 return this;
             } catch (IOException e) {
                 throw new UncheckedIOException("failed to convert source to json", e);
-            }            
+            }
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
-    }   
-    
+    }
+
     public BytesReference mappings() {
         return this.mappings;
     }
@@ -349,8 +349,8 @@ public PutIndexTemplateRequest source(byte[] source, int offset, int length, XCo
      */
     public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) {
         return source(XContentHelper.convertToMap(source, true, xContentType).v2());
-    }    
-    
+    }
+
 
     public Set<Alias> aliases() {
         return this.aliases;
@@ -441,7 +441,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.copyCurrentStructure(parser);
             }
         }
-        
+
         builder.startObject("aliases");
         for (Alias alias : aliases) {
             alias.toXContent(builder, params);

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java

@@ -25,7 +25,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -34,7 +34,7 @@
 import java.util.Objects;
 import java.util.Optional;
 
-public class FindFileStructureRequest implements Validatable, ToXContent {
+public class FindFileStructureRequest implements Validatable, ToXContentFragment {
 
     public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample");
     public static final ParseField TIMEOUT = new ParseField("timeout");

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
+
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -61,6 +62,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.cbor.CborXContent;
@@ -176,7 +178,7 @@ public void testInfo() throws IOException {
         MainResponse testInfo = new MainResponse("nodeName", new MainResponse.Version("number", "buildFlavor", "buildType", "buildHash",
             "buildDate", true, "luceneVersion", "minimumWireCompatibilityVersion", "minimumIndexCompatibilityVersion"),
             "clusterName", "clusterUuid", "You Know, for Search");
-        mockResponse((builder, params) -> {
+        mockResponse((ToXContentFragment) (builder, params) -> {
             // taken from the server side MainResponse
             builder.field("name", testInfo.getNodeName());
             builder.field("cluster_name", testInfo.getClusterName());
@@ -762,12 +764,12 @@ public void testApiNamingConventions() throws Exception {
                     Collectors.mapping(Tuple::v2, Collectors.toSet())));
 
         // TODO remove in 8.0 - we will undeprecate indices.get_template because the current getIndexTemplate
-        // impl will replace the existing getTemplate method. 
+        // impl will replace the existing getTemplate method.
         // The above general-purpose code ignores all deprecated methods which in this case leaves `getTemplate`
-        // looking like it doesn't have a valid implementatation when it does. 
+        // looking like it doesn't have a valid implementatation when it does.
         apiUnsupported.remove("indices.get_template");
-        
-        
+
+
 
         for (Map.Entry<String, Set<Method>> entry : methods.entrySet()) {
             String apiName = entry.getKey();
@@ -830,7 +832,7 @@ private static void assertSyncMethod(Method method, String apiName, List<String>
             assertThat("the return type for method [" + method + "] is incorrect",
                 method.getReturnType().getSimpleName(), equalTo("boolean"));
         } else {
-            // It's acceptable for 404s to be represented as empty Optionals 
+            // It's acceptable for 404s to be represented as empty Optionals
             if (!method.getReturnType().isAssignableFrom(Optional.class)) {
                 assertThat("the return type for method [" + method + "] is incorrect",
                     method.getReturnType().getSimpleName(), endsWith("Response"));

docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc

@@ -12,7 +12,9 @@ It accepts a +{request}+ object and responds with a +{response}+ object.
 [id="{upid}-{api}-request"]
 ==== Stop Data Frame Request
 
-A +{request}+ object requires a non-null `id`.
+A +{request}+ object requires a non-null `id`. `id` can be a comma separated list of Ids
+or a single Id. Wildcards, `*` and `_all` are also accepted.
+
 
 ["source","java",subs="attributes,callouts,macros"]
 ---------------------------------------------------

docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc

@@ -89,7 +89,7 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-=== Intervals
+==== Intervals
 
 The interval of the returned buckets is selected based on the data collected by the 
 aggregation so that the number of buckets returned is less than or equal to the number 

docs/reference/analysis/analyzers/standard-analyzer.asciidoc

@@ -132,7 +132,7 @@ The `standard` analyzer accepts the following parameters:
 `stopwords`::
 
     A pre-defined stop words list like `_english_` or an array  containing a
-    list of stop words.  Defaults to `\_none_`.
+    list of stop words.  Defaults to `_none_`.
 
 `stopwords_path`::
 

docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc

@@ -78,4 +78,4 @@ Elasticsearch provides the following predefined list of languages:
 `_portuguese_`, `_romanian_`, `_russian_`, `_sorani_`, `_spanish_`,
 `_swedish_`, `_thai_`, `_turkish_`.
 
-For the empty stopwords list (to disable stopwords) use: `\_none_`.
+For the empty stopwords list (to disable stopwords) use: `_none_`.

docs/reference/data-frames/apis/stop-transform.asciidoc

@@ -10,9 +10,17 @@ Stops one or more {dataframe-transforms}.
 
 ==== Request
 
-`POST _data_frame/transforms/<data_frame_transform_id>/_stop`
+`POST _data_frame/transforms/<data_frame_transform_id>/_stop` +
 
-//==== Description
+`POST _data_frame/transforms/<data_frame_transform_id1>,<data_frame_transform_id2>/_stop` +
+
+`POST _data_frame/transforms/_all/_stop`
+
+
+==== Description
+You can stop multiple {dataframe-transforms} in a single API request by using a
+comma-separated list of {dataframe-transforms} or a wildcard expression.
+All {dataframe-transforms} can be stopped by using `_all` or `*` as the `<data_frame_transform_id>`.
 
 ==== Path Parameters
 

docs/reference/migration/migrate_8_0.asciidoc

@@ -17,7 +17,11 @@ coming[8.0.0]
 * <<breaking_80_packaging_changes>>
 * <<breaking_80_snapshots_changes>>
 * <<breaking_80_security_changes>>
+* <<breaking_80_ilm_changes>>
 * <<breaking_80_java_changes>>
+* <<breaking_80_network_changes>>
+* <<breaking_80_transport_changes>>
+* <<breaking_80_http_changes>>
 
 //NOTE: The notable-breaking-changes tagged regions are re-used in the
 //Installation and Upgrade Guide
@@ -46,4 +50,8 @@ include::migrate_8_0/mappings.asciidoc[]
 include::migrate_8_0/packaging.asciidoc[]
 include::migrate_8_0/snapshots.asciidoc[]
 include::migrate_8_0/security.asciidoc[]
+include::migrate_8_0/ilm.asciidoc[]
 include::migrate_8_0/java.asciidoc[]
+include::migrate_8_0/network.asciidoc[]
+include::migrate_8_0/transport.asciidoc[]
+include::migrate_8_0/http.asciidoc[]

docs/reference/migration/migrate_8_0/http.asciidoc

@@ -0,0 +1,15 @@
+[float]
+[[breaking_80_http_changes]]
+=== HTTP changes
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+//tag::notable-breaking-changes[]
+
+// end::notable-breaking-changes[]
+
+[float]
+==== Removal of old HTTP settings
+
+The `http.tcp_no_delay` setting was deprecated in 7.x and has been removed in 8.0. It has been replaced by
+`http.tcp.no_delay`.

docs/reference/migration/migrate_8_0/ilm.asciidoc

@@ -0,0 +1,17 @@
+[float]
+[[breaking_80_ilm_changes]]
+=== Index Lifecycle Management changes
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+
+//tag::notable-breaking-changes[]
+
+// end::notable-breaking-changes[]
+
+[float]
+[[ilm-poll-interval-limit]]
+==== `indices.lifecycle.poll_interval` must be greater than 1 second
+
+The setting `indices.lifecycle.poll_interval`, if set too low, can cause
+excessive load on a cluster. This setting must now be set to 1 second or higher.

docs/reference/migration/migrate_8_0/network.asciidoc

@@ -0,0 +1,16 @@
+[float]
+[[breaking_80_network_changes]]
+=== Network changes
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+//tag::notable-breaking-changes[]
+
+// end::notable-breaking-changes[]
+
+[float]
+==== Removal of old network settings
+
+The `network.tcp.connect_timeout` setting was deprecated in 7.x and has been removed in 8.0. This setting
+was a fallback setting for `transport.connect_timeout`. To change the default connection timeout for client
+connections `transport.connect_timeout` should be modified.

docs/reference/migration/migrate_8_0/transport.asciidoc

@@ -0,0 +1,22 @@
+[float]
+[[breaking_80_transport_changes]]
+=== Transport changes
+
+//tag::notable-breaking-changes[]
+[float]
+==== Removal of old transport settings
+
+The following settings have been deprecated in 7.x and removed in 8.0. Each setting has a replacement
+setting that was introduced in 6.7.
+
+- `transport.tcp.port` replaced by `transport.port`
+- `transport.tcp.compress` replaced by `transport.compress`
+- `transport.tcp.connect_timeout` replaced by `transport.connect_timeout`
+- `transport.tcp_no_delay` replaced by `transport.tcp.no_delay`
+- `transport.profiles.profile_name.tcp_no_delay` replaced by `transport.profiles.profile_name.tcp.no_delay`
+- `transport.profiles.profile_name.tcp_keep_alive` replaced by `transport.profiles.profile_name.tcp.keep_alive`
+- `transport.profiles.profile_name.reuse_address` replaced by `transport.profiles.profile_name.tcp.reuse_address`
+- `transport.profiles.profile_name.send_buffer_size` replaced by `transport.profiles.profile_name.tcp.send_buffer_size`
+- `transport.profiles.profile_name.receive_buffer_size` replaced by `transport.profiles.profile_name.tcp.receive_buffer_size`
+
+// end::notable-breaking-changes[]