Merge branch 'main' into ES-6576

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -529,8 +529,8 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
-      - label: "{{matrix.image}} / 8.13.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.2
+      - label: "{{matrix.image}} / 8.13.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -543,7 +543,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.2
+          BWC_VERSION: 8.13.3
 
       - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0

.buildkite/pipelines/periodic.yml

@@ -312,16 +312,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.2
-      - label: 8.13.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.2#bwcTest
+      - label: 8.13.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.2
+          BWC_VERSION: 8.13.3
       - label: 8.14.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.0#bwcTest
         timeout_in_minutes: 300
@@ -396,7 +396,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -438,7 +438,7 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -30,5 +30,5 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.4"
   - "8.12.2"
-  - "8.13.2"
+  - "8.13.3"
   - "8.14.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.20"
-  - "8.13.2"
+  - "8.13.3"
   - "8.14.0"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -62,13 +62,13 @@ public class MrjarPlugin implements Plugin<Project> {
     public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
-        var isIdea = System.getProperty("idea.active", "false").equals("true");
+        var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true");
         var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true");
 
         // Ignore version-specific source sets if we are importing into IntelliJ and have not explicitly enabled this.
         // Avoids an IntelliJ bug:
         // https://youtrack.jetbrains.com/issue/IDEA-285640/Compiler-Options-Settings-language-level-is-set-incorrectly-with-JDK-19ea
-        if (isIdea == false || ideaSourceSetsEnabled) {
+        if (isIdeaSync == false || ideaSourceSetsEnabled) {
             List<Integer> mainVersions = findSourceVersions(project);
             List<String> mainSourceSets = new ArrayList<>();
             mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java

@@ -457,6 +457,26 @@ public void testLegacyV3() throws GeneralSecurityException, IOException {
         assertThat(toByteArray(wrapper.getFile("file_setting")), equalTo("file_value".getBytes(StandardCharsets.UTF_8)));
     }
 
+    public void testLegacyV5() throws GeneralSecurityException, IOException {
+        final Path configDir = createTempDir();
+        final Path keystore = configDir.resolve("elasticsearch.keystore");
+        try (
+            InputStream is = KeyStoreWrapperTests.class.getResourceAsStream("/format-v5-with-password-elasticsearch.keystore");
+            OutputStream os = Files.newOutputStream(keystore)
+        ) {
+            final byte[] buffer = new byte[4096];
+            int readBytes;
+            while ((readBytes = is.read(buffer)) > 0) {
+                os.write(buffer, 0, readBytes);
+            }
+        }
+        final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir);
+        assertNotNull(wrapper);
+        wrapper.decrypt("keystorepassword".toCharArray());
+        assertThat(wrapper.getFormatVersion(), equalTo(5));
+        assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed")));
+    }
+
     public void testSerializationNewlyCreated() throws Exception {
         final KeyStoreWrapper wrapper = KeyStoreWrapper.create();
         wrapper.setString("string_setting", "string_value".toCharArray());

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.env.Environment;
 
 import java.io.InputStream;
@@ -46,8 +47,20 @@ public void testKeystoreUpgradeV4() throws Exception {
         assertKeystoreUpgrade("/format-v4-elasticsearch.keystore", KeyStoreWrapper.V4_VERSION);
     }
 
+    public void testKeystoreUpgradeV5() throws Exception {
+        assertKeystoreUpgradeWithPassword("/format-v5-with-password-elasticsearch.keystore", KeyStoreWrapper.LE_VERSION);
+    }
+
     private void assertKeystoreUpgrade(String file, int version) throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
+        assertKeystoreUpgrade(file, version, null);
+    }
+
+    private void assertKeystoreUpgradeWithPassword(String file, int version) throws Exception {
+        assertKeystoreUpgrade(file, version, "keystorepassword");
+    }
+
+    private void assertKeystoreUpgrade(String file, int version, @Nullable String password) throws Exception {
         final Path keystore = KeyStoreWrapper.keystorePath(env.configFile());
         try (InputStream is = KeyStoreWrapperTests.class.getResourceAsStream(file); OutputStream os = Files.newOutputStream(keystore)) {
             is.transferTo(os);
@@ -56,11 +69,17 @@ private void assertKeystoreUpgrade(String file, int version) throws Exception {
             assertNotNull(beforeUpgrade);
             assertThat(beforeUpgrade.getFormatVersion(), equalTo(version));
         }
+        if (password != null) {
+            terminal.addSecretInput(password);
+            terminal.addSecretInput(password);
+        }
         execute();
+        terminal.reset();
+
         try (KeyStoreWrapper afterUpgrade = KeyStoreWrapper.load(env.configFile())) {
             assertNotNull(afterUpgrade);
             assertThat(afterUpgrade.getFormatVersion(), equalTo(KeyStoreWrapper.CURRENT_VERSION));
-            afterUpgrade.decrypt(new char[0]);
+            afterUpgrade.decrypt(password != null ? password.toCharArray() : new char[0]);
             assertThat(afterUpgrade.getSettingNames(), hasItem(KeyStoreWrapper.SEED_SETTING.getKey()));
         }
     }
@@ -69,5 +88,4 @@ public void testKeystoreDoesNotExist() {
         final UserException e = expectThrows(UserException.class, this::execute);
         assertThat(e, hasToString(containsString("keystore not found at [" + KeyStoreWrapper.keystorePath(env.configFile()) + "]")));
     }
-
 }

docs/changelog/103374.yaml

@@ -0,0 +1,16 @@
+pr: 103374
+summary: Cut over stored fields to ZSTD for compression
+area: Search
+type: enhancement
+issues: []
+highlight:
+  title: Stored fields are now compressed with ZStandard instead of LZ4/DEFLATE
+  body: |-
+    Stored fields are now compressed by splitting documents into blocks, which
+    are then compressed independently with ZStandard. `index.codec: default`
+    (default) uses blocks of at most 14kB or 128 documents compressed with level
+    0, while `index.codec: best_compression` uses blocks of at most 240kB or
+    2048 documents compressed at level 3. On most datasets that we tested
+    against, this yielded storage improvements in the order of 10%, slightly
+    faster indexing and similar retrieval latencies.
+  notable: true

docs/changelog/106077.yaml

@@ -0,0 +1,7 @@
+pr: 106077
+summary: Fix merging component templates with a mix of dotted and nested object mapper
+  definitions
+area: Mapping
+type: bug
+issues:
+ - 105482

docs/changelog/107007.yaml

@@ -0,0 +1,5 @@
+pr: 107007
+summary: "ESQL: Support ST_DISJOINT"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/107107.yaml

@@ -0,0 +1,5 @@
+pr: 107107
+summary: Increase KDF iteration count in `KeyStoreWrapper`
+area: Infra/CLI
+type: enhancement
+issues: []

docs/changelog/107122.yaml

@@ -0,0 +1,5 @@
+pr: 107122
+summary: Avoid unintentionally clearing the `DataStream.rolloverOnWrite` flag
+area: Data streams
+type: bug
+issues: []

docs/changelog/107129.yaml

@@ -0,0 +1,5 @@
+pr: 107129
+summary: Track ongoing search tasks
+area: Search
+type: enhancement
+issues: []

docs/changelog/107158.yaml

@@ -0,0 +1,5 @@
+pr: 107158
+summary: "ESQL: allow sorting by expressions and not only regular fields"
+area: ES|QL
+type: feature
+issues: []

docs/changelog/107183.yaml

@@ -0,0 +1,5 @@
+pr: 107183
+summary: ES|QL fix no-length substring with supplementary (4-byte) character
+area: ES|QL
+type: bug
+issues: []

docs/reference/cluster/stats.asciidoc

@@ -1821,7 +1821,8 @@ The API returns the following response:
                 "all_in_bytes": 0,
                 "coordinating_rejections": 0,
                 "primary_rejections": 0,
-                "replica_rejections": 0
+                "replica_rejections": 0,
+                "primary_document_rejections": 0
             },
             "limit" : "0b",
             "limit_in_bytes": 0

docs/reference/data-streams/data-streams.asciidoc

@@ -18,6 +18,28 @@ automate the management of these backing indices. For example, you can use
 hardware and delete unneeded indices. {ilm-init} can help you reduce costs and
 overhead as your data grows.
 
+
+[discrete]
+[[should-you-use-a-data-stream]]
+== Should you use a data stream?
+
+To determine whether you should use a data stream for your data, you should consider the format of
+the data, and your expected interaction. A good candidate for using a data stream will match the
+following criteria:
+
+* Your data contains a timestamp field, or one could be automatically generated.
+* You mostly perform indexing requests, with occasional updates and deletes.
+* You index documents without an `_id`, or when indexing documents with an explicit `_id` you expect first-write-wins behavior.
+
+For most time series data use-cases, a data stream will be a good fit. However, if you find that
+your data doesn't fit into these categories (for example, if you frequently send multiple documents
+using the same `_id` expecting last-write-wins), you may want to use an index alias with a write
+index instead. See documentation for <<manage-time-series-data-without-data-streams,managing time
+series data without a data stream>> for more information.
+
+Keep in mind that some features such as <<tsds,Time Series Data Streams (TSDS)>> and
+<<data-stream-lifecycle,data stream lifecycles>> require a data stream.
+
 [discrete]
 [[backing-indices]]
 == Backing indices
@@ -116,19 +138,19 @@ You should not derive any intelligence from the backing indices names.
 
 [discrete]
 [[data-streams-append-only]]
-== Append-only
+== Append-only (mostly)
 
-Data streams are designed for use cases where existing data is rarely,
-if ever, updated. You cannot send update or deletion requests for existing
-documents directly to a data stream. Instead, use the
+Data streams are designed for use cases where existing data is rarely updated. You cannot send
+update or deletion requests for existing documents directly to a data stream. However, you can still
+<<update-delete-docs-in-a-backing-index,update or delete documents>> in a data stream by submitting
+requests directly to the document's backing index.
+
+If you need to update a larger number of documents in a data stream, you can use the
 <<update-docs-in-a-data-stream-by-query,update by query>> and
 <<delete-docs-in-a-data-stream-by-query,delete by query>> APIs.
 
-If needed, you can <<update-delete-docs-in-a-backing-index,update or delete
-documents>> by submitting requests directly to the document's backing index.
-
-TIP: If you frequently update or delete existing time series data, use an index
-alias with a write index instead of a data stream. See
+TIP: If you frequently send multiple documents using the same `_id` expecting last-write-wins, you
+may want to use an index alias with a write index instead. See
 <<manage-time-series-data-without-data-streams>>.
 
 include::set-up-a-data-stream.asciidoc[]

docs/reference/docs/bulk.asciidoc

@@ -414,9 +414,7 @@ This parameter is only returned for successful actions.
 `result`::
 (string)
 Result of the operation. Successful values are `created`, `deleted`, and
-`updated`.
-+
-This parameter is only returned for successful operations.
+`updated`. Other valid values are `noop` and `not_found`.
 
 `_shards`::
 (object)