Merge branch 'main' into mv-sort

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -30,6 +30,9 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -56,24 +59,41 @@ public void apply(Project project) {
         var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
 
         var srcDir = project.getProjectDir().toPath().resolve("src");
+        List<Integer> mainVersions = new ArrayList<>();
         try (var subdirStream = Files.list(srcDir)) {
             for (Path sourceset : subdirStream.toList()) {
                 assert Files.isDirectory(sourceset);
                 String sourcesetName = sourceset.getFileName().toString();
                 Matcher sourcesetMatcher = MRJAR_SOURCESET_PATTERN.matcher(sourcesetName);
                 if (sourcesetMatcher.matches()) {
-                    int javaVersion = Integer.parseInt(sourcesetMatcher.group(1));
-                    addMrjarSourceset(project, javaExtension, sourcesetName, javaVersion);
+                    mainVersions.add(Integer.parseInt(sourcesetMatcher.group(1)));
                 }
             }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
+
+        Collections.sort(mainVersions);
+        List<String> parentSourceSets = new ArrayList<>();
+        parentSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
+        for (int javaVersion : mainVersions) {
+            String sourcesetName = "main" + javaVersion;
+            addMrjarSourceset(project, javaExtension, sourcesetName, parentSourceSets, javaVersion);
+            parentSourceSets.add(sourcesetName);
+        }
     }
 
-    private void addMrjarSourceset(Project project, JavaPluginExtension javaExtension, String sourcesetName, int javaVersion) {
+    private void addMrjarSourceset(
+        Project project,
+        JavaPluginExtension javaExtension,
+        String sourcesetName,
+        List<String> parentSourceSets,
+        int javaVersion
+    ) {
         SourceSet sourceSet = javaExtension.getSourceSets().maybeCreate(sourcesetName);
-        GradleUtils.extendSourceSet(project, SourceSet.MAIN_SOURCE_SET_NAME, sourcesetName);
+        for (String parentSourceSetName : parentSourceSets) {
+            GradleUtils.extendSourceSet(project, parentSourceSetName, sourcesetName);
+        }
 
         var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
         jarTask.configure(task -> {

docs-mdx/painless/painless-field-context.mdx

@@ -0,0 +1,136 @@
+---
+id: enElasticsearchPainlessPainlessFieldContext
+slug: /en/elasticsearch/painless/painless-field-context
+title: Field context
+description: Description to be written
+tags: []
+---
+
+<div id="painless-field-context"></div>
+
+Use a Painless script to create a
+[script field](((ref))/search-fields.html#script-fields) to return
+a customized value for each document in the results of a query.
+
+**Variables**
+
+`params` (`Map`, read-only)
+    : User-defined parameters passed in as part of the query.
+
+`doc` (`Map`, read-only)
+    : Contains the fields of the specified document where each field is a
+    `List` of values.
+
+[`params['_source']`](((ref))/mapping-source-field.html) (`Map`, read-only)
+    : Contains extracted JSON in a `Map` and `List` structure for the fields
+    existing in a stored document.
+
+**Return**
+
+`Object`
+    : The customized value for each document.
+
+**API**
+
+Both the standard <DocLink id="enElasticsearchPainlessPainlessApiReferenceShared">Painless API</DocLink> and
+<DocLink id="enElasticsearchPainlessPainlessApiReferenceField">Specialized Field API</DocLink> are available.
+
+**Example**
+
+To run this example, first follow the steps in
+<DocLink id="enElasticsearchPainlessPainlessContextExamples">context examples</DocLink>.
+
+You can then use these two example scripts to compute custom information
+for each search hit and output it to two new fields.
+
+The first script gets the doc value for the `datetime` field and calls
+the `getDayOfWeekEnum` function to determine the corresponding day of the week.
+
+```Painless
+doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)
+```
+
+The second script calculates the number of actors. Actors' names are stored
+as a keyword array in the `actors` field.
+
+```Painless
+doc['actors'].size()   [^1]
+```
+[^1]: By default, doc values are not available for `text` fields. If `actors` was
+a `text` field, you could still calculate the number of actors by extracting
+values from `_source` with `params['_source']['actors'].size()`.
+
+The following request returns the calculated day of week and the number of
+actors that appear in each play:
+
+```console
+GET seats/_search
+{
+  "size": 2,
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "day-of-week": {
+      "script": {
+        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
+      }
+    },
+    "number-of-actors": {
+      "script": {
+        "source": "doc['actors'].size()"
+      }
+    }
+  }
+}
+```
+{/* TEST[setup:seats] */}
+
+```console-result
+{
+  "took" : 68,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 11,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "seats",
+        "_id" : "1",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            4
+          ]
+        }
+      },
+      {
+        "_index" : "seats",
+        "_id" : "2",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            1
+          ]
+        }
+      }
+    ]
+  }
+}
+```
+{/* TESTRESPONSE[s/"took" : 68/"took" : "$body.took"/] */}

docs/changelog/105393.yaml

@@ -0,0 +1,5 @@
+pr: 105393
+summary: Adding support for hex-encoded byte vectors on knn-search
+area: Vector Search
+type: feature
+issues: []

docs/changelog/105442.yaml

@@ -0,0 +1,6 @@
+pr: 105442
+summary: Handling exceptions on watcher reload
+area: Watcher
+type: bug
+issues:
+ - 69842

docs/changelog/105470.yaml

@@ -0,0 +1,5 @@
+pr: 105470
+summary: Add retrievers using the parser-only approach
+area: Ranking
+type: enhancement
+issues: []

docs/changelog/105894.yaml

@@ -0,0 +1,5 @@
+pr: 105894
+summary: Add allocation stats
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/105941.yaml

@@ -0,0 +1,5 @@
+pr: 105941
+summary: Field caps performance pt2
+area: Search
+type: enhancement
+issues: []

docs/changelog/106094.yaml

@@ -0,0 +1,5 @@
+pr: 106094
+summary: "ESQL: Support partially folding CASE"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106150.yaml

@@ -0,0 +1,5 @@
+pr: 106150
+summary: Use correct system index bulk executor
+area: CRUD
+type: bug
+issues: []

docs/changelog/106171.yaml

@@ -0,0 +1,6 @@
+pr: 106171
+summary: Do not log error on node restart when the transform is already failed
+area: Transform
+type: enhancement
+issues:
+ - 106168

docs/changelog/106172.yaml

@@ -0,0 +1,5 @@
+pr: 106172
+summary: "[Profiling] Allow to override index settings"
+area: Application
+type: enhancement
+issues: []

docs/changelog/106189.yaml

@@ -0,0 +1,6 @@
+pr: 106189
+summary: Fix numeric sorts in `_cat/nodes`
+area: CAT APIs
+type: bug
+issues:
+ - 48070

docs/changelog/106247.yaml

@@ -0,0 +1,5 @@
+pr: 106247
+summary: Fix a downsample persistent task assignment bug
+area: Downsampling
+type: bug
+issues: []

docs/changelog/97561.yaml

@@ -0,0 +1,5 @@
+pr: 97561
+summary: Add index forecasts to /_cat/allocation output
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/99048.yaml

@@ -0,0 +1,6 @@
+pr: 99048
+summary: String sha512() painless function
+area: Infra/Scripting
+type: enhancement
+issues:
+  - 97691

docs/reference/analysis/normalizers.asciidoc

@@ -6,15 +6,15 @@ token. As a consequence, they do not have a tokenizer and only accept a subset
 of the available char filters and token filters. Only the filters that work on
 a per-character basis are allowed. For instance a lowercasing filter would be
 allowed, but not a stemming filter, which needs to look at the keyword as a
-whole. The current list of filters that can be used in a normalizer is
-following: `arabic_normalization`, `asciifolding`, `bengali_normalization`,
+whole. The current list of filters that can be used in a normalizer definition
+are: `arabic_normalization`, `asciifolding`, `bengali_normalization`,
 `cjk_width`, `decimal_digit`, `elision`, `german_normalization`,
 `hindi_normalization`, `indic_normalization`, `lowercase`, `pattern_replace`,
 `persian_normalization`, `scandinavian_folding`, `serbian_normalization`,
 `sorani_normalization`, `trim`, `uppercase`.
 
 Elasticsearch ships with a `lowercase` built-in normalizer. For other forms of
-normalization a custom configuration is required.
+normalization, a custom configuration is required.
 
 [discrete]
 === Custom normalizers

docs/reference/api-conventions.asciidoc

@@ -337,8 +337,22 @@ value `true`. All other values will raise an error.
 [discrete]
 === Number Values
 
-All REST APIs support providing numbered parameters as `string` on top
-of supporting the native JSON number types.
+When passing a numeric parameter in a request body, you may use a `string`
+containing the number instead of the native numeric type. For example:
+
+[source,console]
+--------------------------------------------------
+POST /_search
+{
+  "size": "1000"
+}
+--------------------------------------------------
+
+Integer-valued fields in a response body are described as `integer` (or
+occasionally `long`) in this manual, but there are generally no explicit bounds
+on such values. JSON, SMILE, CBOR and YAML all permit arbitrarily large integer
+values. Do not assume that `integer` fields in a response body will always fit
+into a 32-bit signed integer.
 
 [[byte-units]]
 [discrete]

docs/reference/cat/allocation.asciidoc

@@ -57,6 +57,16 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v]
 `shards`::
 Number of primary and replica shards assigned to the node.
 
+`shards.undesired`::
+Amount of shards that are scheduled to be moved elsewhere in the cluster
+or -1 other than desired balance allocator is used
+
+`write_load.forecast`::
+Sum of index write load forecasts
+
+`disk.indices.forecast`::
+Sum of shard size forecasts
+
 `disk.indices`::
 Disk space used by the node's shards. Does not include disk space for the
 <<index-modules-translog,translog>> or unassigned shards.
@@ -99,6 +109,8 @@ IP address and port for the node.
 `node`::
 Name for the node. Set using <<node-name,`node.name`>>.
 
+`node.role`, `r`, `role`, `nodeRole`::
+Node roles
 
 [[cat-allocation-api-example]]
 ==== {api-examples-title}
@@ -113,8 +125,8 @@ The API returns the following response:
 
 [source,txt]
 --------------------------------------------------
-shards disk.indices disk.used disk.avail disk.total disk.percent host      ip        node    node.role
-     1         260b    47.3gb     43.4gb    100.7gb           46 127.0.0.1 127.0.0.1 CSUXak2 himrst
+shards shards.undesired write_load.forecast disk.indices.forecast disk.indices disk.used disk.avail disk.total disk.percent host      ip        node    node.role
+     1                0                 0.0                  260b         260b    47.3gb     43.4gb    100.7gb           46 127.0.0.1 127.0.0.1 CSUXak2 himrst
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/ s/46/\\d+/]
 // TESTRESPONSE[s/CSUXak2 himrst/.+/ non_json]