Merge branch '8.17' into backport/8.17/pr-118691

carlosdelest · Dec 18, 2024 · 19630a6 · 19630a6
2 parents 0512be8 + 414e291
commit 19630a6
Show file tree

Hide file tree

Showing 31 changed files with 170 additions and 33 deletions.
diff --git a/.backportrc.json b/.backportrc.json
@@ -4,7 +4,7 @@
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.18.0$" : "8.x",
+    "^v8.17.1$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
-}
+}
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.27", "8.16.2", "8.17.1"]
+            BWC_VERSION: ["7.17.27", "8.16.3", "8.17.1"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -575,8 +575,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
+      - label: "{{matrix.image}} / 8.16.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -589,7 +589,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.2
+          BWC_VERSION: 8.16.3
 
       - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -648,8 +648,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.2#bwcTest
+      - label: 8.16.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -658,7 +658,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.2
+          BWC_VERSION: 8.16.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -752,7 +752,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.27", "8.16.2", "8.17.1"]
+            BWC_VERSION: ["7.17.27", "8.16.3", "8.17.1"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -800,7 +800,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.27", "8.16.2", "8.17.1"]
+            BWC_VERSION: ["7.17.27", "8.16.3", "8.17.1"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -33,5 +33,5 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.2"
+  - "8.16.3"
   - "8.17.1"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.27"
-  - "8.16.2"
+  - "8.16.3"
   - "8.17.1"
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -24,7 +24,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:1b51ff6dba78c98d3e02b0cd64a8ce3238c7a40408d21e3af12a329d44db6f23",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bfdeddb33330a281950c2a54adef991dbbe6a42832bc505d13b11beaf50ae73f",
         "-wolfi",
         "apk"
     ),

diff --git a/docs/changelog/118380.yaml b/docs/changelog/118380.yaml
diff --git a/docs/changelog/118837.yaml b/docs/changelog/118837.yaml
@@ -0,0 +1,5 @@
+pr: 118837
+summary: Add missing timeouts to rest-api-spec ILM APIs
+area: "ILM+SLM"
+type: bug
+issues: []
diff --git a/docs/changelog/118844.yaml b/docs/changelog/118844.yaml
@@ -0,0 +1,5 @@
+pr: 118844
+summary: Add missing timeouts to rest-api-spec ingest APIs
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/reference/alias.asciidoc b/docs/reference/alias.asciidoc
@@ -407,3 +407,24 @@ POST _aliases
 }
 ----
 // TEST[s/^/PUT my-index-2099.05.06-000001\n/]
+
+[discrete]
+[[remove-index]]
+=== Remove an index
+
+To remove an index, use the aliases API's `remove_index` action.
+
+[source,console]
+----
+POST _aliases
+{
+  "actions": [
+    {
+      "remove_index": {
+        "index": "my-index-2099.05.06-000001"
+      }
+    }
+  ]
+}
+----
+// TEST[s/^/PUT my-index-2099.05.06-000001\n/]
diff --git a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc
@@ -24,7 +24,7 @@ and retention requirements.
 You want to send log files to an {es} cluster so you can visualize and analyze
 the data. This data has the following retention requirements:
 
-* When the write index reaches 50GB or is 30 days old, roll over to a new index.
+* When the primary shard size of the write index reaches 50GB or the index is 30 days old, roll over to a new index.
 * After rollover, keep indices in the hot data tier for 30 days.
 * 30 days after rollover:
 ** Move indices to the warm data tier.
@@ -84,7 +84,7 @@ To save the `logs@lifecycle` policy as a new policy in {kib}:
 . On the **Edit policy logs** page, toggle **Save as new policy**, and then provide a new name for the policy, for example, `logs-custom`.
 
 The `logs@lifecycle` policy uses the recommended rollover defaults: Start writing to a new
-index when the current write index reaches 50GB or becomes 30 days old.
+index when the primary shard size of the current write index reaches 50GB or the index becomes 30 days old.
 
 To view or change the rollover settings, click **Advanced settings** for the hot
 phase. Then disable **Use recommended defaults** to display the rollover

diff --git a/docs/reference/images/elastic-rerank.png b/docs/reference/images/elastic-rerank.png
diff --git a/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png b/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png
diff --git a/docs/reference/migration/migrate_8_17.asciidoc b/docs/reference/migration/migrate_8_17.asciidoc
@@ -9,9 +9,6 @@ your application to {es} 8.17.
 
 See also <<release-highlights>> and <<es-release-notes>>.
 
-coming::[8.17.0]
-
-
 [discrete]
 [[breaking-changes-8.17]]
 === Breaking changes

diff --git a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
@@ -138,8 +138,8 @@ normal priority deployments.
 Controls how many inference requests are allowed in the queue at a time.
 Every machine learning node in the cluster where the model can be allocated
 has a queue of this size; when the number of requests exceeds the total value,
-new requests are rejected with a 429 error. Defaults to 1024. Max allowed value
-is 1000000.
+new requests are rejected with a 429 error. Defaults to 10000. Max allowed value
+is 100000.
 
 `threads_per_allocation`::
 (Optional, integer)
@@ -173,7 +173,7 @@ The API returns the following results:
             "model_bytes": 265632637,
             "threads_per_allocation" : 1,
             "number_of_allocations" : 1,
-            "queue_capacity" : 1024,
+            "queue_capacity" : 10000,
             "priority": "normal"
         },
         "routing_table": {
@@ -229,4 +229,4 @@ POST _ml/trained_models/my_model/deployment/_start?deployment_id=my_model_for_se
   }
 }
 --------------------------------------------------
-// TEST[skip:TBD]
+// TEST[skip:TBD]
diff --git a/docs/reference/release-notes/8.17.0.asciidoc b/docs/reference/release-notes/8.17.0.asciidoc
@@ -1,6 +1,11 @@
 [[release-notes-8.17.0]]
 == {es} version 8.17.0
 
+[IMPORTANT]
+====
+The 8.17.0 release contains a fix for a potential security vulnerability. See our https://discuss.elastic.co/c/announcements/security-announcements/31[security advisory] for more details.
+====
+
 Also see <<breaking-changes-8.17,Breaking changes in 8.17>>.
 
 [[license-8.17.0]]

diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc
@@ -15,6 +15,31 @@ The `logsdb` index mode has been GA-ed. This feature allows Elasticsearch to sto
 `logsdb` index mode allows to reduce storage usage up to ~3 times compared to storing logs in Elasticsearch using the default index mode.
 Refer to the <<logs-data-stream,documentation>> for further information.
 
+[discrete]
+[[elastic-rerank]]
+=== Elastic Rerank
+In 8.17, we are introducing {ml-docs}/ml-nlp-rerank.html[Elastic Rerank], Elastic’s new model for semantic reranking. Reranking can be used as the second stage on top of your existing search approach, whether this is lexical search, ELSER, or dense embeddings, resulting in significant semantic relevance gains mainly via recall improvements. This is very important for RAG applications, where you want to send the most relevant information to an LLM in order to generate the possible responses.
+
+Elastic Rerank is a cross-encoder architecture and it is in technical preview in 8.17.
+
+image::images/elastic-rerank.png[Diagram showing how document text is modeled for a cross encoder, width=90%]
+
+To understand reranking and how you can use it to improve your search, read https://www.elastic.co/search-labs/blog/elastic-semantic-reranker-part-1[What is semantic reranking and how to use it], a blog by the creators of Elastic Rerank.
+
+For more information on the Elastic Rerank model architecture and how it compares for relevance and cost against other reranking models, see https://www.elastic.co/search-labs/blog/elastic-semantic-reranker-part-2[Introducing Elastic Rerank] by the same authors.
+
+[discrete]
+[[pre-configured-elser-endpoint]]
+=== Pre-configured default ELSER endpoint
+With this release, we are introducing a pre-configured default ELSER endpoint. This removes the need for any inference configuration in order to use ELSER. There is no more need for an inference endpoint setup or for downloading the model and configuring threads and allocations through the trained models UI. Simply create a <<semantic-text,semantic text>> mapping with your indices and start using it. The default inference endpoint will download, deploy and start using ELSER for inference with  {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] out of the box.
+
+[discrete]
+[[full-text-search-esql]]
+=== Full-text search for ES|QL
+
+The technical preview of the new `MATCH` and `QSTR` (query string) functions in ES|QL makes log searches easier and more intuitive. `MATCH` delivers full-text search functionality in ES|QL using a Lucene match query, while `QTSR` aids in more advanced filtering of log data by enabling Lucene query string queries. 
+
+Full-text search for ES|QL enables easier and more performant searching in Discover, especially when dealing with multiple terms or conditional logic. Learn more in the <<esql-search-functions, ES|QL full-text search documentation>>.
 
 // Add previous release to the list
 Other versions:

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.delete_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.delete_lifecycle.json
@@ -25,6 +25,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.explain_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.explain_lifecycle.json
@@ -33,6 +33,10 @@
       "only_errors": {
         "type": "boolean",
         "description": "filters the indices included in the response to ones in an ILM error state, implies only_managed"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
       }
     }
   }

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.get_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.get_lifecycle.json
@@ -31,6 +31,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.put_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.put_lifecycle.json
@@ -26,7 +26,16 @@
         }
       ]
     },
-    "params":{},
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    },
     "body":{
       "description":"The lifecycle policy definition to register"
     }

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.start.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.start.json
@@ -19,6 +19,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.stop.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.stop.json
@@ -19,6 +19,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json
@@ -26,6 +26,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json
@@ -26,6 +26,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json
@@ -27,6 +27,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     },
     "body":{
       "description":"The database configuration definition",

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json
@@ -27,6 +27,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     },
     "body":{
       "description":"The database configuration definition",