Merge branch 'master' into feature/apm-integration

.ci/jobs.t/elastic+elasticsearch+pull-request+cloud-deploy.yml

@@ -0,0 +1,50 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+cloud-deploy"
+    display-name: "elastic / elasticsearch - pull request cloud-deploy"
+    description: "Testing of Elasticsearch pull requests - cloud-deploy"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+cloud-deploy"
+    node: "general-purpose && docker"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/cloud-deploy.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/cloud-deploy
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'cloud-deploy'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh buildCloudDockerImage
+      - shell: |
+          #!/bin/bash
+          set +x
+          VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+          export VAULT_TOKEN
+          export DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/elasticsearch-ci/prod_docker_registry_credentials)"
+          export DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/elasticsearch-ci/prod_docker_registry_credentials)"
+          export ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}')
+          export DOCKER_TAG=docker.elastic.co/elasticsearch-ci/elasticsearch-cloud:${ES_VERSION}-${ghprbActualCommit:0:7}
+          docker tag elasticsearch-cloud:test $DOCKER_TAG
+          echo $DOCKER_REGISTRY_PASSWORD | docker login -u $DOCKER_REGISTRY_USERNAME --password-stdin docker.elastic.co
+          unset VAULT_TOKEN DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD
+          set -x
+          docker push $DOCKER_TAG

build-tools-internal/src/main/resources/changelog-schema.json

@@ -191,33 +191,18 @@
     "breakingArea": {
       "type": "string",
       "enum": [
-        "API",
-        "Aggregation",
-        "Allocation",
-        "Authentication",
-        "CCR",
-        "Cluster",
-        "Discovery",
-        "Engine",
-        "HTTP",
-        "Highlighters",
-        "Indices",
-        "Java",
-        "License Information",
+        "Cluster and node setting",
+        "Command line tool",
+        "Index setting",
+        "JVM option",
+        "Java API",
         "Logging",
-        "Machine Learning",
-        "Mappings",
-        "Networking",
+        "Mapping",
         "Packaging",
-        "Plugins",
-        "Script Cache",
-        "Search Changes",
-        "Search",
-        "Security",
-        "Settings",
-        "Snapshot and Restore",
-        "Transform",
-        "Transport"
+        "Painless",
+        "REST API",
+        "System requirement",
+        "Transform"
       ]
     },
     "additionalProperties": false

build-tools-internal/src/main/resources/checkstyle.xml

@@ -114,6 +114,7 @@
         <property name="ignoreSetter" value="true" />
         <property name="setterCanReturnItsClass" value="true"/>
         <property name="ignoreFormat" value="^(threadPool)$"/>
+        <property name="ignoreAbstractMethods" value="true"/>
         <message key="hidden.field" value="''{0}'' hides a field." />
     </module>
     -->

build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java

@@ -144,6 +144,7 @@ private static void configureDependencies(final Project project) {
         var dependencies = project.getDependencies();
         dependencies.add("compileOnly", "org.elasticsearch:elasticsearch:" + VersionProperties.getElasticsearch());
         dependencies.add("testImplementation", "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch());
+        dependencies.add("testImplementation", "org.apache.logging.log4j:log4j-core:" + VersionProperties.getVersions().get("log4j"));
 
         // we "upgrade" these optional deps to provided for plugins, since they will run
         // with a full elasticsearch server that includes optional deps

docs/Versions.asciidoc

@@ -68,7 +68,3 @@ ifeval::["{source_branch}"=="7.x"]
 :apm-server-ref-v:     {apm-server-ref-m}
 :apm-overview-ref-v:   {apm-overview-ref-m}
 endif::[]
-
-// Max recommended snapshots in a snapshot repo.
-// Used in the snapshot/restore docs.
-:max-snapshot-count: 200

docs/changelog/78968.yaml

@@ -5,7 +5,7 @@ type: breaking
 issues: []
 breaking:
   title: HTTP Status code has changed for the Cluster Health API in case of a server timeout
-  area: API
+  area: REST API
   details: |-
     The cluster health API includes options for waiting
     for certain health conditions to be satisfied. If the requested conditions are

docs/plugins/plugin-script.asciidoc

@@ -1,7 +1,7 @@
 [[plugin-management]]
 == Plugin Management
 
-The `plugin` script is used to install, list, and remove plugins. It is
+Use the `elasticsearch-plugin` command line tool to install, list, and remove plugins. It is
 located in the `$ES_HOME/bin` directory by default but it may be in a
 different location depending on which Elasticsearch package you installed:
 
@@ -26,6 +26,13 @@ Otherwise run `bin/elasticsearch-plugin` as the user that owns all of the Elasti
 files.
 =====================
 
+[discrete]
+[[plugin-management-docker]]
+=== Docker
+
+If you run {es} using Docker, you can manage plugins using a
+<<manage-plugins-using-configuration-file,configuration file>>.
+
 [[installation]]
 === Installing Plugins
 
@@ -279,3 +286,52 @@ The default location of the `plugins` directory depends on which package you ins
 * {ref}/zip-windows.html#windows-layout[Directory layout of Windows `.zip` archives]
 * {ref}/deb.html#deb-layout[Directory layout of Debian package]
 * {ref}/rpm.html#rpm-layout[Directory layout of RPM]
+
+
+[[manage-plugins-using-configuration-file]]
+=== Manage plugins using a configuration file
+
+[IMPORTANT]
+.Docker only
+=====================
+This feature is only available for https://www.docker.elastic.co/[official {es}
+Docker images]. Other {es} distributions will not start with a
+plugin configuration file.
+=====================
+
+If you run {es} using Docker, you can manage plugins using a declarative configuration file.
+When {es} starts up, it will compare the plugins in the file with those
+that are currently installed, and add or remove plugins as required. {es}
+will also upgrade offical plugins when you upgrade {es} itself.
+
+The file is called `elasticsearch-plugins.yml`, and must be placed in the
+Elasticsearch configuration directory, alongside `elasticsearch.yml`. Here
+is an example:
+
+[source,yaml]
+----
+plugins:
+  - id: analysis-icu
+  - id: repository-azure
+  - id: custom-mapper
+    location: https://example.com/archive/custom-mapper-1.0.0.zip
+----
+
+This example installs the official `analysis-icu` and
+`repository-azure` plugins, and one unofficial plugin. Every plugin must provide
+an `id`. Unofficial plugins must also provide a `location`. This is
+typically a URL, but Maven coordinates are also supported. The downloaded
+plugin's name must match the ID in the configuration file.
+
+While {es} will respect the
+https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html[standard
+Java proxy system properties] when downloading plugins, you can also configure an
+HTTP proxy to use explicitly in the configuration file. For example:
+
+[source,yaml]
+----
+plugins:
+  - id: custom-mapper
+    location: https://example.com/archive/custom-mapper-1.0.0.zip
+proxy: proxy.example.com:8443
+----

docs/reference/cluster/update-settings.asciidoc

@@ -48,9 +48,15 @@ PUT /_cluster/settings
 
 An example of a transient update:
 
-// tag::transient-settings-deprecation[]
-deprecated::[7.16,Transient settings are deprecated and will be removed in a future release. Use persistent cluster settings instead. See the <<transient-settings-migration-guide>>.]
-// end::transient-settings-deprecation[]
+// tag::transient-settings-warning[]
+[WARNING]
+====
+We no longer recommend using transient cluster settings. Use persistent cluster
+settings instead. If a cluster becomes unstable, transient settings can clear
+unexpectedly, resulting in a potentially undesired cluster configuration. See
+the <<transient-settings-migration-guide>>.
+====
+// end::transient-settings-warning[]
 
 [source,console]
 --------------------------------------------------

docs/reference/getting-started.asciidoc

@@ -17,7 +17,6 @@ The simplest way to set up {es} is to create a managed deployment with {ess} on
 {ecloud}. If you prefer to manage your own test environment, you can install and
 run {es} using Docker.
 
-include::{es-repo-dir}/tab-widgets/code.asciidoc[]
 include::{es-repo-dir}/tab-widgets/quick-start-install-widget.asciidoc[]
 
 [discrete]

docs/reference/how-to/fix-common-cluster-issues.asciidoc

@@ -302,7 +302,6 @@ exceeds 85%.
 
 **Check JVM memory pressure**
 
-include::{es-repo-dir}/tab-widgets/code.asciidoc[]
 include::{es-repo-dir}/tab-widgets/jvm-memory-pressure-widget.asciidoc[]
 
 **Check garbage collection logs**

docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc

@@ -26,7 +26,6 @@ stream.
 
 The steps for setting up data tiers vary based on your deployment type:
 
-include::{es-repo-dir}/tab-widgets/code.asciidoc[]
 include::{es-repo-dir}/tab-widgets/data-tiers-widget.asciidoc[]
 
 [discrete]

docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc

@@ -87,8 +87,9 @@ Once created, you cannot update
 or index documents to an enrich index.
 Instead, update your source indices
 and <<execute-enrich-policy-api,execute>> the enrich policy again.
-This creates a new enrich index from your updated source indices
-and deletes the previous enrich index.
+This creates a new enrich index from your updated source indices. 
+The previous enrich index will deleted with a delayed maintenance job. 
+By default this is done every 15 minutes. 
 // end::update-enrich-index[]
 
 Because this API request performs several operations,

docs/reference/migration/index.asciidoc

@@ -1,9 +1,6 @@
 [[breaking-changes]]
 = Migration guide
 
-:ess-setting-change: {ess-icon} indicates a change to a supported {cloud}/ec-add-user-settings.html[user setting] for {ess}.
-:ess-skip-section: If you use {ess}, skip this section. {ess} handles these changes for you.
-
 This section discusses the changes that you need to be aware of to migrate
 your application to {version}. For more information about what's new in this
 release, see the <<release-highlights>> and <<es-release-notes>>.

docs/reference/migration/migrate_8_0.asciidoc

@@ -27,17 +27,17 @@ the old behavior is supported until the next major release.
 To find out if you are using any deprecated functionality,
 enable <<deprecation-logging, deprecation logging>>.
 
-include::migrate_8_0/_cluster-node-setting-changes.asciidoc[]
-include::migrate_8_0/_command-line-tool-changes.asciidoc[]
-include::migrate_8_0/_index-setting-changes.asciidoc[]
-include::migrate_8_0/_java-api-changes.asciidoc[]
-include::migrate_8_0/_jvm-option-changes.asciidoc[]
-include::migrate_8_0/_logging-changes.asciidoc[]
-include::migrate_8_0/_mapping-changes.asciidoc[]
-include::migrate_8_0/_packaging-changes.asciidoc[]
-include::migrate_8_0/_painless-changes.asciidoc[]
-include::migrate_8_0/_rest-api-changes.asciidoc[]
-include::migrate_8_0/_system-req-changes.asciidoc[]
+include::migrate_8_0/cluster-node-setting-changes.asciidoc[]
+include::migrate_8_0/command-line-tool-changes.asciidoc[]
+include::migrate_8_0/index-setting-changes.asciidoc[]
+include::migrate_8_0/java-api-changes.asciidoc[]
+include::migrate_8_0/jvm-option-changes.asciidoc[]
+include::migrate_8_0/logging-changes.asciidoc[]
+include::migrate_8_0/mapping-changes.asciidoc[]
+include::migrate_8_0/packaging-changes.asciidoc[]
+include::migrate_8_0/painless-changes.asciidoc[]
+include::migrate_8_0/rest-api-changes.asciidoc[]
+include::migrate_8_0/system-req-changes.asciidoc[]
 include::migrate_8_0/transform.asciidoc[]
 
 [discrete]
@@ -63,22 +63,19 @@ enable <<deprecation-logging, deprecation logging>>.
 ==== Cluster and node setting deprecations
 
 [[deprecate-transient-cluster-settings]]
-.Transient cluster settings are deprecated.
+.We no longer recommend using transient cluster settings.
 [%collapsible]
 ====
 *Details* +
-Starting in 7.16, transient cluster settings are deprecated and will be removed in a future release. This is because transient
-cluster settings have an unpredictable lifecycle. Transient cluster settings do not survive full cluster restarts or
-cluster instability. In these cases, {es} recovers the cluster state
-from persistent storage, effectively erasing the transient settings. The loss of transient settings can happen
-unexpectedly, leading to a potentially undesired cluster configuration.
+We no longer recommend using transient cluster settings. Use persistent cluster
+settings instead. If a cluster becomes unstable, transient settings can clear
+unexpectedly, resulting in an undesired cluster configuration.
 
 *Impact* +
-To avoid deprecation warnings, discontinue use of transient settings when modifying
-your cluster settings through the `PUT _cluster/settings` REST API. Use persistent
-settings instead. See the
+Transient cluster settings are not yet deprecated, but we plan to deprecate them
+in a future release. For migration steps, see the
 {ref}/transient-settings-migration-guide.html[Transient settings migration
-guide]. 
+guide].
 ====
 
 [discrete]