Merge remote-tracking branch 'upstream/master' into compat/register_c…

…ompatible_handlers

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java

@@ -62,10 +62,10 @@ static Provider<RestIntegTestTask> registerTask(Project project, SourceSet sourc
             project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
                 Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
                 testTask.dependsOn(bundle);
-                if (project.getPath().contains("modules:")) {
+                if (project.getPath().contains("modules:") || project.getPath().startsWith(":x-pack:plugin")) {
                     testTask.getClusters().forEach(c -> c.module(bundle.getArchiveFile()));
                 } else {
-                    testTask.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())));
+                    testTask.getClusters().forEach(c -> c.plugin(bundle.getArchiveFile()));
                 }
             });
         });

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -617,27 +617,23 @@ private void copyExtraJars() {
     }
 
     private void installModules() {
-        if (testDistribution == TestDistribution.INTEG_TEST) {
-            logToProcessStdout("Installing " + modules.size() + "modules");
-            for (Provider<File> module : modules) {
-                Path destination = getDistroDir().resolve("modules")
-                    .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", ""));
-                // only install modules that are not already bundled with the integ-test distribution
-                if (Files.exists(destination) == false) {
-                    fileSystemOperations.copy(spec -> {
-                        if (module.get().getName().toLowerCase().endsWith(".zip")) {
-                            spec.from(archiveOperations.zipTree(module));
-                        } else if (module.get().isDirectory()) {
-                            spec.from(module);
-                        } else {
-                            throw new IllegalArgumentException("Not a valid module " + module + " for " + this);
-                        }
-                        spec.into(destination);
-                    });
-                }
+        logToProcessStdout("Installing " + modules.size() + "modules");
+        for (Provider<File> module : modules) {
+            Path destination = getDistroDir().resolve("modules")
+                .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", ""));
+            // only install modules that are not already bundled with the integ-test distribution
+            if (Files.exists(destination) == false) {
+                fileSystemOperations.copy(spec -> {
+                    if (module.get().getName().toLowerCase().endsWith(".zip")) {
+                        spec.from(archiveOperations.zipTree(module));
+                    } else if (module.get().isDirectory()) {
+                        spec.from(module);
+                    } else {
+                        throw new IllegalArgumentException("Not a valid module " + module + " for " + this);
+                    }
+                    spec.into(destination);
+                });
             }
-        } else {
-            LOGGER.info("Not installing " + modules.size() + "(s) since the " + distributions + " distribution already has them");
         }
     }
 

distribution/build.gradle

@@ -335,7 +335,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 
   dependencies {
     libs project(':server')
-    libs project(':libs:elasticsearch-plugin-classloader')
     libs project(':distribution:tools:java-version-checker')
     libs project(':distribution:tools:launchers')
 

docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc

@@ -30,7 +30,7 @@ analysis chain matches a predicate.
 `token.type` (`String`, read-only)::
         The type of the current token
 
-`token.keyword` ('boolean`, read-only)::
+`token.keyword` (`boolean`, read-only)::
         Whether or not the current token is marked as a keyword
 
 *Return*
@@ -40,4 +40,4 @@ analysis chain matches a predicate.
 
 *API*
 
-The standard <<painless-api-reference-shared, Painless API>> is available.
+The standard <<painless-api-reference-shared, Painless API>> is available.

docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc

@@ -61,3 +61,20 @@ The API returns the following result:
   "acknowledged": true
 }
 --------------------------------------------------
+
+This example deletes all autoscaling policies.
+
+[source,console]
+--------------------------------------------------
+DELETE /_autoscaling/policy/*
+--------------------------------------------------
+// TEST
+
+The API returns the following result:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------

docs/reference/cat/indices.asciidoc

@@ -77,7 +77,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segme
 `local`::
 (Optional, boolean)
 +
-deprecated::[7.10.0,"This parameter does not affect the request. It will be removed in a future release."]
+deprecated::[7.11.0,"This parameter does not affect the request. It will be removed in a future release."]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 

docs/reference/cat/shards.asciidoc

@@ -276,7 +276,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help]
 `local`::
 (Optional, boolean)
 +
-deprecated::[7.10.0,"This parameter does not affect the request. It will be removed in a future release."]
+deprecated::[7.11.0,"This parameter does not affect the request. It will be removed in a future release."]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 

docs/reference/index-modules/allocation/data_tier_allocation.asciidoc

@@ -19,7 +19,7 @@ for data tier filtering.
 
 [discrete]
 [[data-tier-allocation-filters]]
-====Data tier allocation settings
+==== Data tier allocation settings
 
 
 `index.routing.allocation.include._tier`::
@@ -43,7 +43,6 @@ for data tier filtering.
     Assign the index to the first tier in the list that has an available node.
     This prevents indices from remaining unallocated if no nodes are available
     in the preferred tier.
-
     For example, if you set `index.routing.allocation.include._tier_preference`
     to `data_warm,data_hot`, the index is allocated to the warm tier if there
     are nodes with the `data_warm` role. If there are no nodes in the warm tier,

docs/reference/indices/put-component-template.asciidoc

@@ -5,7 +5,7 @@
 ++++
 
 Creates or updates a component template.
-Component templates are building blocks for constructing <<indices-templates,index templates>>.
+Component templates are building blocks for constructing <<indices-templates,index templates>>
 that specify index <<mapping,mappings>>, <<index-modules-settings,settings>>,
 and <<indices-aliases,aliases>>.
 

docs/reference/ingest/apis/get-pipeline.asciidoc

@@ -45,8 +45,12 @@ GET /_ingest/pipeline/my-pipeline-id
 [[get-pipeline-api-path-params]]
 ==== {api-path-parms-title}
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=path-pipeline]
-
+`<pipeline>`::
+(Optional, string)
+Comma-separated list of pipeline IDs to retrieve. Wildcard (`*`) expressions are
+supported.
++
+To get all ingest pipelines, omit this parameter or use `*`.
 
 
 [[get-pipeline-api-query-params]]

docs/reference/modules/node.asciidoc

@@ -277,14 +277,6 @@ To create a dedicated ingest node, set:
 node.roles: [ ingest ]
 ----
 
-[[node-ingest-node-setting]]
-// tag::node-ingest-tag[]
-`node.ingest` {ess-icon}::
-Determines whether a node is an ingest node. <<ingest,Ingest nodes>> can apply
-an ingest pipeline to transform and enrich a document before indexing. Default:
-`true`.
-// end::node-ingest-tag[]
-
 [[coordinating-only-node]]
 ==== Coordinating only node
 

docs/reference/rest-api/common-parms.asciidoc

@@ -642,12 +642,6 @@ The number of search or bulk index operations processed. Documents are
 processed in batches instead of individually.
 end::pages-processed[]
 
-tag::path-pipeline[]
-`<pipeline>`::
-(Optional, string) Comma-separated list or wildcard expression of pipeline IDs
-used to limit the request.
-end::path-pipeline[]
-
 tag::pivot[]
 The method for transforming the data. These objects define the pivot function
 `group by` fields and the aggregation to reduce the data.

docs/reference/search/suggesters/phrase-suggest.asciidoc

@@ -149,7 +149,7 @@ The response contains suggestions scored by the most likely spelling correction
     `gram_size` is set to the `max_shingle_size` if not explicitly set.
 
 `real_word_error_likelihood`::
-    The likelihood of a term being a
+    The likelihood of a term being
     misspelled even if the term exists in the dictionary. The default is
     `0.95`, meaning 5% of the real words are misspelled.
 

docs/reference/setup/important-settings/path-settings.asciidoc

@@ -2,32 +2,32 @@
 [discrete]
 === Path settings
 
-If you are using the `.zip` or `.tar.gz` archives, the `data` and `logs`
-directories are sub-folders of `$ES_HOME`. If these important folders are left
-in their default locations, there is a high risk of them being deleted while
-upgrading {es} to a new version.
-
-In production use, you will almost certainly want to change the locations of the
-`path.data` and `path.logs` folders:
-
-[source,yaml]
---------------------------------------------------
-path:
-  logs: /var/log/elasticsearch
-  data: /var/data/elasticsearch
---------------------------------------------------
-
-The RPM and Debian distributions already use custom paths for `data` and `logs`.
-
-The `path.data` settings can be set to multiple paths, in which case all paths
-will be used to store data. However, the files belonging to a single shard will
-all be stored on the same data path:
-
-[source,yaml]
---------------------------------------------------
-path:
-  data:
-    - /mnt/elasticsearch_1
-    - /mnt/elasticsearch_2
-    - /mnt/elasticsearch_3
---------------------------------------------------
+For <<targz,macOS `.tar.gz`>>, <<targz,Linux `.tar.gz`>>, and
+<<zip-windows,Windows `.zip`>> installations, {es} writes data and logs to the
+respective `data` and `logs` subdirectories of `$ES_HOME` by default.
+However, files in `$ES_HOME` risk deletion during an upgrade.
+
+In production, we strongly recommend you set the `path.data` and `path.logs` in
+`elasticsearch.yml` to locations outside of `$ES_HOME`.
+
+TIP: <<docker,Docker>>, <<deb,Debian>>, <<rpm,RPM>>, <<brew,macOS Homebrew>>,
+and <<windows,Windows `.msi`>> installations write data and log to locations
+outside of `$ES_HOME` by default.
+
+Supported `path.data` and `path.logs` values vary by platform:
+
+include::{es-repo-dir}/tab-widgets/code.asciidoc[]
+
+include::{es-repo-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[]
+
+If needed, you can specify multiple paths in `path.data`. {es} stores the node's
+data across all provided paths but keeps each shard's data on the same path.
+
+WARNING: {es} does not balance shards across a node's data paths. High disk
+usage in a single path can trigger a <<disk-based-shard-allocation,high disk
+usage watermark>> for the entire node. If triggered, {es} will not add shards to
+the node, even if the node’s other paths have available disk space. If you need
+additional disk space, we recommend you add a new node rather than additional
+data paths. 
+
+include::{es-repo-dir}/tab-widgets/multi-data-path-widget.asciidoc[]

docs/reference/snapshot-restore/apis/put-repo-api.asciidoc

@@ -150,11 +150,11 @@ plugins:
 +
 --
 (Required, object)
-Contains settings for the repository. Valid properties for the `settings` object
-depend on the repository type, set using the
-<<put-snapshot-repo-api-request-type,`type`>> parameter.
+Contains settings for the repository.
 
-.Valid `settings` properties for `fs` repositories
+The following `settings` properties are valid for all repository types:
+
+.Properties of `settings`
 [%collapsible%open]
 ====
 `chunk_size`::
@@ -168,11 +168,13 @@ file size).
 If `true`, metadata files, such as index mappings and settings, are compressed
 in snapshots. Data files are not compressed. Defaults to `true`.
 
-`location`::
-(Required, string)
-Location of the shared filesystem used to store and retrieve snapshots. This
-location must be registered in the `path.repo` setting on all master and data
-nodes in the cluster.
+`max_number_of_snapshots`::
+(Optional, integer)
+Maximum number of snapshots the repository can contain. Defaults to `500`.
++
+WARNING: We do not recommend increasing `max_number_of_snapshots`. Larger
+snapshot repositories may degrade master node performance and cause stability
+issues. Instead, delete older snapshots or use multiple repositories.
 
 `max_restore_bytes_per_sec`::
 (Optional, <<byte-units,byte value>>)
@@ -206,6 +208,19 @@ the repository but not create snapshots in it.
 =====
 ====
 
+Other accepted `settings` properties depend on the repository type, set using the
+<<put-snapshot-repo-api-request-type,`type`>> parameter.
+
+.Valid `settings` properties for `fs` repositories
+[%collapsible%open]
+====
+`location`::
+(Required, string)
+Location of the shared filesystem used to store and retrieve snapshots. This
+location must be registered in the `path.repo` setting on all master and data
+nodes in the cluster.
+====
+
 .Valid `settings` properties for `source` repositories
 [%collapsible%open]
 ====

docs/reference/snapshot-restore/register-repository.asciidoc

@@ -100,71 +100,20 @@ are left untouched and in place.
 [[snapshots-filesystem-repository]]
 === Shared file system repository
 
-The shared file system repository (`"type": "fs"`) uses the shared file system to store snapshots. In order to register
-the shared file system repository it is necessary to mount the same shared filesystem to the same location on all
-master and data nodes. This location (or one of its parent directories) must be registered in the `path.repo`
-setting on all master and data nodes.
+Use a shared file system repository (`"type": "fs"`) to store snapshots on a
+shared file system.
 
-Assuming that the shared filesystem is mounted to `/mount/backups/my_fs_backup_location`, the following setting should
-be added to `elasticsearch.yml` file:
+To register a shared file system repository, first mount the file system to the
+same location on all master and data nodes. Then add the file system's
+path or parent directory to the `path.repo` setting in `elasticsearch.yml` for
+each master and data node. For running clusters, this requires a
+<<restart-cluster-rolling,rolling restart>> of each node.
 
-[source,yaml]
---------------
-path.repo: ["/mount/backups", "/mount/longterm_backups"]
---------------
-
-The `path.repo` setting supports Microsoft Windows UNC paths as long as at least server name and share are specified as
-a prefix and back slashes are properly escaped:
-
-[source,yaml]
---------------
-path.repo: ["\\\\MY_SERVER\\Snapshots"]
---------------
-
-After all nodes are restarted, the following command can be used to register the shared file system repository with
-the name `my_fs_backup`:
-
-[source,console]
------------------------------------
-PUT /_snapshot/my_fs_backup
-{
-  "type": "fs",
-  "settings": {
-    "location": "/mount/backups/my_fs_backup_location",
-    "compress": true
-  }
-}
------------------------------------
-// TEST[skip:no access to absolute path]
-
-If the repository location is specified as a relative path this path will be resolved against the first path specified
-in `path.repo`:
+Supported `path.repo` values vary by platform:
 
-[source,console]
------------------------------------
-PUT /_snapshot/my_fs_backup
-{
-  "type": "fs",
-  "settings": {
-    "location": "my_fs_backup_location",
-    "compress": true
-  }
-}
------------------------------------
-// TEST[continued]
+include::{es-repo-dir}/tab-widgets/code.asciidoc[]
 
-The following settings are supported:
-
-`location`:: Location of the snapshots. Mandatory.
-`compress`:: Turns on compression of the snapshot files. Compression is applied only to metadata files (index mapping and settings). Data files are not compressed. Defaults to `true`.
-`chunk_size`:: Big files can be broken down into chunks during snapshotting if needed. Specify the chunk size as a value and
-unit, for example: `1GB`, `10MB`, `5KB`, `500B`. Defaults to `null` (unlimited chunk size).
-`max_restore_bytes_per_sec`:: Throttles per node restore rate. Defaults to unlimited. Note that restores are also throttled through <<recovery,recovery settings>>.
-`max_snapshot_bytes_per_sec`:: Throttles per node snapshot rate. Defaults to `40mb` per second.
-`readonly`:: Makes repository read-only.  Defaults to `false`.
-`max_number_of_snapshots`:: Limits the maximum number of snapshots that the repository may contain. Defaults to `500`. Note that snapshot repositories do not
-scale indefinitely in size and might lead to master node performance and stability issues if they grow past a certain size. We do not recommend increasing this setting.
-Instead you should delete older snapshots or use multiple repositories.
+include::{es-repo-dir}/tab-widgets/register-fs-repo-widget.asciidoc[]
 
 [discrete]
 [[snapshots-read-only-repository]]