Merge branch 'main' into add-bwcLucene87Codec

cbuescher · Nov 28, 2024 · 4a46d80 · 4a46d80
2 parents da3db30 + 5d68697
commit 4a46d80
Show file tree

Hide file tree

Showing 100 changed files with 2,665 additions and 662 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -47,6 +47,8 @@ export GRADLE_BUILD_CACHE_PASSWORD
 BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
 export BUILDKITE_API_TOKEN
 
+export GH_TOKEN="$VAULT_GITHUB_TOKEN"
+
 if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
 
@@ -117,3 +119,5 @@ if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then
   echo "$(hostname -i | cut -d' ' -f 2)  $(hostname -f)." | sudo tee /etc/dnsmasq.hosts
   sudo systemctl restart dnsmasq.service
 fi
+
+.buildkite/scripts/get-latest-test-mutes.sh
diff --git a/.buildkite/hooks/pre-command.bat b/.buildkite/hooks/pre-command.bat
@@ -15,9 +15,12 @@ set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
 set COMPOSE_HTTP_TIMEOUT=120
 set JOB_BRANCH=%BUILDKITE_BRANCH%
 
+set GH_TOKEN=%VAULT_GITHUB_TOKEN%
+
 set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 
 bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
+bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
 
 exit /b 0
diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -273,8 +273,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
+      - label: "{{matrix.image}} / 8.15.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -287,7 +287,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.6
 
       - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
+      - label: 8.15.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.6
         retry:
           automatic:
             - exit_status: "-1"

diff --git a/.buildkite/scripts/get-latest-test-mutes.sh b/.buildkite/scripts/get-latest-test-mutes.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+  exit 0
+fi
+
+testMuteBranch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
+testMuteFile="$(mktemp)"
+
+# If this PR contains changes to muted-tests.yml, we disable this functionality
+# Otherwise, we wouldn't be able to test unmutes
+if [[ ! $(gh pr diff "$BUILDKITE_PULL_REQUEST" --name-only | grep 'muted-tests.yml') ]]; then
+  gh api -H 'Accept: application/vnd.github.v3.raw' "repos/elastic/elasticsearch/contents/muted-tests.yml?ref=$testMuteBranch" > "$testMuteFile"
+
+  if [[ -s "$testMuteFile" ]]; then
+    mkdir -p ~/.gradle
+    # This is using gradle.properties instead of an env var so that it's easily compatible with the Windows pre-command hook
+    echo "org.gradle.project.org.elasticsearch.additional.muted.tests=$testMuteFile" >> ~/.gradle/gradle.properties
+  fi
+fi
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -14,7 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.4"
+  - "8.15.6"
   - "8.16.2"
   - "8.17.0"
   - "8.18.0"

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
@@ -77,7 +77,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        new PluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
+        PluginsLoader.createPluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java
@@ -28,10 +28,12 @@
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 
 public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
-    private final List<String> excludePatterns = new ArrayList<>();
+    private final Set<String> excludePatterns = new LinkedHashSet<>();
     private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
 
     public MutedTestsBuildService() {
@@ -43,23 +45,23 @@ public MutedTestsBuildService() {
         }
     }
 
-    public List<String> getExcludePatterns() {
+    public Set<String> getExcludePatterns() {
         return excludePatterns;
     }
 
-    private List<String> buildExcludePatterns(File file) {
+    private Set<String> buildExcludePatterns(File file) {
         List<MutedTest> mutedTests;
 
         try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
             mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
             if (mutedTests == null) {
-                return Collections.emptyList();
+                return Collections.emptySet();
             }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
 
-        List<String> excludes = new ArrayList<>();
+        Set<String> excludes = new LinkedHashSet<>();
         if (mutedTests.isEmpty() == false) {
             for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
                 if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) {

diff --git a/docs/changelog/114193.yaml b/docs/changelog/114193.yaml
diff --git a/docs/changelog/114227.yaml b/docs/changelog/114227.yaml
diff --git a/docs/changelog/114268.yaml b/docs/changelog/114268.yaml
diff --git a/docs/changelog/114317.yaml b/docs/changelog/114317.yaml
@@ -0,0 +1,5 @@
+pr: 114317
+summary: "ESQL: CATEGORIZE as a `BlockHash`"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114521.yaml b/docs/changelog/114521.yaml
diff --git a/docs/changelog/114548.yaml b/docs/changelog/114548.yaml
diff --git a/docs/changelog/116277.yaml b/docs/changelog/116277.yaml
diff --git a/docs/changelog/116292.yaml b/docs/changelog/116292.yaml
diff --git a/docs/changelog/116357.yaml b/docs/changelog/116357.yaml
diff --git a/docs/changelog/116382.yaml b/docs/changelog/116382.yaml
diff --git a/docs/changelog/116408.yaml b/docs/changelog/116408.yaml
diff --git a/docs/changelog/116478.yaml b/docs/changelog/116478.yaml
diff --git a/docs/changelog/116650.yaml b/docs/changelog/116650.yaml
diff --git a/docs/changelog/116676.yaml b/docs/changelog/116676.yaml
diff --git a/docs/changelog/116915.yaml b/docs/changelog/116915.yaml
diff --git a/docs/changelog/116918.yaml b/docs/changelog/116918.yaml
diff --git a/docs/changelog/116942.yaml b/docs/changelog/116942.yaml
diff --git a/docs/changelog/116995.yaml b/docs/changelog/116995.yaml
diff --git a/docs/changelog/117182.yaml b/docs/changelog/117182.yaml
diff --git a/docs/changelog/117235.yaml b/docs/changelog/117235.yaml
diff --git a/docs/changelog/117265.yaml b/docs/changelog/117265.yaml
@@ -0,0 +1,5 @@
+pr: 117265
+summary: Async search responses have CCS metadata while searches are running
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/reference/esql/functions/kibana/definition/categorize.json b/docs/reference/esql/functions/kibana/definition/categorize.json
diff --git a/docs/reference/esql/functions/types/categorize.asciidoc b/docs/reference/esql/functions/types/categorize.asciidoc
diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc
@@ -69,15 +69,15 @@ include::inference-shared.asciidoc[tag=service-settings]
 These settings are specific to the `elasticsearch` service.
 --
 
-`adaptive_allocations`:::
-(Optional, object)
-include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
-
 `deployment_id`:::
 (Optional, string)
 The `deployment_id` of an existing trained model deployment.
 When `deployment_id` is used the `model_id` is optional.
 
+`adaptive_allocations`:::
+(Optional, object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
+
 `enabled`::::
 (Optional, Boolean)
 include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-enabled]
@@ -119,7 +119,6 @@ include::inference-shared.asciidoc[tag=task-settings]
 Returns the document instead of only the index. Defaults to `true`.
 =====
 
-
 [discrete]
 [[inference-example-elasticsearch-elser]]
 ==== ELSER via the `elasticsearch` service
@@ -137,7 +136,7 @@ PUT _inference/sparse_embedding/my-elser-model
     "adaptive_allocations": { <1>
       "enabled": true,
       "min_number_of_allocations": 1,
-      "max_number_of_allocations": 10
+      "max_number_of_allocations": 4
     },
     "num_threads": 1,
     "model_id": ".elser_model_2" <2>
@@ -150,6 +149,34 @@ PUT _inference/sparse_embedding/my-elser-model
 Valid values are `.elser_model_2` and `.elser_model_2_linux-x86_64`.
 For further details, refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation].
 
+[discrete]
+[[inference-example-elastic-reranker]]
+==== Elastic Rerank via the `elasticsearch` service
+
+The following example shows how to create an {infer} endpoint called `my-elastic-rerank` to perform a `rerank` task type using the built-in Elastic Rerank cross-encoder model.
+
+The API request below will automatically download the Elastic Rerank model if it isn't already downloaded and then deploy the model.
+Once deployed, the model can be used for semantic re-ranking with a <<text-similarity-reranker-retriever-example-elastic-rerank,`text_similarity_reranker` retriever>>.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/rerank/my-elastic-rerank
+{
+  "service": "elasticsearch",
+  "service_settings": {
+    "model_id": ".rerank-v1", <1>
+    "num_threads": 1,
+    "adaptive_allocations": { <2>
+      "enabled": true,
+      "min_number_of_allocations": 1,
+      "max_number_of_allocations": 4
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The `model_id` must be the ID of the built-in Elastic Rerank model: `.rerank-v1`.
+<2> {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[Adaptive allocations] will be enabled with the minimum of 1 and the maximum of 10 allocations. 
 
 [discrete]
 [[inference-example-elasticsearch]]
@@ -186,7 +213,7 @@ If using the Python client, you can set the `timeout` parameter to a higher valu
 
 [discrete]
 [[inference-example-eland]]
-==== Models uploaded by Eland via the elasticsearch service
+==== Models uploaded by Eland via the `elasticsearch` service
 
 The following example shows how to create an {infer} endpoint called
 `my-msmarco-minilm-model` to perform a `text_embedding` task type.

diff --git a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc
@@ -235,7 +235,7 @@ The reason for the current state. Usually only populated when the `routing_state
 (string)
 The current routing state.
 --
-* `starting`: The model is attempting to allocate on this model, inference calls are not yet accepted.
+* `starting`: The model is attempting to allocate on this node, inference calls are not yet accepted.
 * `started`: The model is allocated and ready to accept inference requests.
 * `stopping`: The model is being deallocated from this node.
 * `stopped`: The model is fully deallocated from this node.