Merge branch 'main' into esql-javadocs-for-new-data-types

.buildkite/scripts/dra-workflow.sh

@@ -75,6 +75,7 @@ find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \;
 
 echo --- Running release-manager
 
+set +e
 # Artifacts should be generated
 docker run --rm \
   --name release-manager \
@@ -91,4 +92,16 @@ docker run --rm \
   --version "$ES_VERSION" \
   --artifact-set main \
   --dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
-  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json"
+  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
+2>&1 | tee release-manager.log
+EXIT_CODE=$?
+set -e
+
+# This failure is just generating a ton of noise right now, so let's just ignore it
+# This should be removed once this issue has been fixed
+if grep "elasticsearch-ubi-9.0.0-SNAPSHOT-docker-image.tar.gz" release-manager.log; then
+  echo "Ignoring error about missing ubi artifact"
+  exit 0
+fi
+
+exit "$EXIT_CODE"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:973431347ad45f40e01afbbd010bf9de929c088a63382239b90dd84f39618bc8",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:55b297da5151d2a2997e8ab9729fe1304e4869389d7090ab7031cc29530f69f8",
         "-wolfi",
         "apk"
     ),

docs/changelog/115020.yaml

@@ -0,0 +1,5 @@
+pr: 115020
+summary: Adding endpoint creation validation for all task types to remaining services
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/116739.yaml

@@ -0,0 +1,5 @@
+pr: 116739
+summary: Change default Docker image to be based on UBI minimal instead of Ubuntu
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/117404.yaml

@@ -0,0 +1,5 @@
+pr: 117404
+summary: Correct bit * byte and bit * float script comparisons
+area: Vector Search
+type: bug
+issues: []

docs/reference/connector/docs/connectors-content-syncs.asciidoc

@@ -52,7 +52,7 @@ However, a fast, accessible third-party data source that stores huge amounts of
 
 [NOTE]
 ====
-Incremental syncs for the SharePoint Online connector use specific logic.
+Incremental syncs for <<es-connectors-sharepoint-online,SharePoint Online>> and <<es-connectors-google-drive,Google Drive>> connectors use specific logic.
 All other connectors use the same shared connector framework logic for incremental syncs.
 ====
 

docs/reference/data-streams/tsds-reindex.asciidoc

@@ -202,7 +202,7 @@ POST /_component_template/destination_template
 POST /_index_template/2
 {
   "index_patterns": [
-    "k8s*"
+    "k9s*"
   ],
   "composed_of": [
     "destination_template"

docs/reference/esql/functions/spatial-functions.asciidoc

@@ -8,19 +8,19 @@
 {esql} supports these spatial functions:
 
 // tag::spatial_list[]
-* experimental:[] <<esql-st_intersects>>
-* experimental:[] <<esql-st_disjoint>>
-* experimental:[] <<esql-st_contains>>
-* experimental:[] <<esql-st_within>>
-* experimental:[] <<esql-st_x>>
-* experimental:[] <<esql-st_y>>
-* experimental:[] <<esql-st_distance>>
+* <<esql-st_distance>>
+* <<esql-st_intersects>>
+* <<esql-st_disjoint>>
+* <<esql-st_contains>>
+* <<esql-st_within>>
+* <<esql-st_x>>
+* <<esql-st_y>>
 // end::spatial_list[]
 
+include::layout/st_distance.asciidoc[]
 include::layout/st_intersects.asciidoc[]
 include::layout/st_disjoint.asciidoc[]
 include::layout/st_contains.asciidoc[]
 include::layout/st_within.asciidoc[]
 include::layout/st_x.asciidoc[]
 include::layout/st_y.asciidoc[]
-include::layout/st_distance.asciidoc[]

docs/reference/geospatial-analysis.asciidoc

@@ -38,11 +38,11 @@ Data is often messy and incomplete. <<ingest,Ingest pipelines>> lets you clean,
 
 <<esql,ES|QL>> has support for <<esql-spatial-functions, Geospatial Search>> functions, enabling efficient index searching for documents that intersect with, are within, are contained by, or are disjoint from a query geometry. In addition, the `ST_DISTANCE` function calculates the distance between two points.
 
-* experimental:[] <<esql-st_intersects>>
-* experimental:[] <<esql-st_disjoint>>
-* experimental:[] <<esql-st_contains>>
-* experimental:[] <<esql-st_within>>
-* experimental:[] <<esql-st_distance>>
+* <<esql-st_intersects>>
+* <<esql-st_disjoint>>
+* <<esql-st_contains>>
+* <<esql-st_within>>
+* <<esql-st_distance>>
 
 [discrete]
 [[geospatial-aggregate]]

docs/reference/intro.asciidoc

@@ -85,7 +85,7 @@ You can deploy {es} in various ways.
 **Hosted options**
 
 * {cloud}/ec-getting-started-trial.html[*Elastic Cloud Hosted*]: {es} is available as part of the hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14-day free trial].
-* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
+* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless*]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
 
 **Advanced options**
 

docs/reference/vectors/vector-functions.asciidoc

@@ -336,6 +336,10 @@ When using `bit` vectors, not all the vector functions are available. The suppor
 this is the sum of the bitwise AND of the two vectors. If providing `float[]` or `byte[]`, who has `dims` number of elements, as a query vector, the `dotProduct` is
 the sum of the floating point values using the stored `bit` vector as a mask.
 
+NOTE: When comparing `floats` and `bytes` with `bit` vectors, the `bit` vector is treated as a mask in big-endian order.
+For example, if the `bit` vector is `10100001` (e.g. the single byte value `161`) and its compared
+with array of values `[1, 2, 3, 4, 5, 6, 7, 8]` the `dotProduct` will be `1 + 3 + 8 = 16`.
+
 Here is an example of using dot-product with bit vectors.
 
 [source,console]

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java

@@ -51,6 +51,8 @@ public static long ipByteBinByte(byte[] q, byte[] d) {
     /**
      * Compute the inner product of two vectors, where the query vector is a byte vector and the document vector is a bit vector.
      * This will return the sum of the query vector values using the document vector as a mask.
+     * When comparing the bits with the bytes, they are done in "big endian" order. For example, if the byte vector
+     * is [1, 2, 3, 4, 5, 6, 7, 8] and the bit vector is [0b10000000], the inner product will be 1.0.
      * @param q the query vector
      * @param d the document vector
      * @return the inner product of the two vectors
@@ -63,9 +65,9 @@ public static int ipByteBit(byte[] q, byte[] d) {
         // now combine the two vectors, summing the byte dimensions where the bit in d is `1`
         for (int i = 0; i < d.length; i++) {
             byte mask = d[i];
-            for (int j = 0; j < Byte.SIZE; j++) {
+            for (int j = Byte.SIZE - 1; j >= 0; j--) {
                 if ((mask & (1 << j)) != 0) {
-                    result += q[i * Byte.SIZE + j];
+                    result += q[i * Byte.SIZE + Byte.SIZE - 1 - j];
                 }
             }
         }
@@ -75,6 +77,8 @@ public static int ipByteBit(byte[] q, byte[] d) {
     /**
      * Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a bit vector.
      * This will return the sum of the query vector values using the document vector as a mask.
+     * When comparing the bits with the floats, they are done in "big endian" order. For example, if the float vector
+     * is [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] and the bit vector is [0b10000000], the inner product will be 1.0.
      * @param q the query vector
      * @param d the document vector
      * @return the inner product of the two vectors
@@ -86,9 +90,9 @@ public static float ipFloatBit(float[] q, byte[] d) {
         float result = 0;
         for (int i = 0; i < d.length; i++) {
             byte mask = d[i];
-            for (int j = 0; j < Byte.SIZE; j++) {
+            for (int j = Byte.SIZE - 1; j >= 0; j--) {
                 if ((mask & (1 << j)) != 0) {
-                    result += q[i * Byte.SIZE + j];
+                    result += q[i * Byte.SIZE + Byte.SIZE - 1 - j];
                 }
             }
         }

libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java

@@ -21,6 +21,22 @@ public class ESVectorUtilTests extends BaseVectorizationTests {
     static final ESVectorizationProvider defaultedProvider = BaseVectorizationTests.defaultProvider();
     static final ESVectorizationProvider defOrPanamaProvider = BaseVectorizationTests.maybePanamaProvider();
 
+    public void testIpByteBit() {
+        byte[] q = new byte[16];
+        byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) };
+        random().nextBytes(q);
+        int expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15];
+        assertEquals(expected, ESVectorUtil.ipByteBit(q, d));
+    }
+
+    public void testIpFloatBit() {
+        float[] q = new float[16];
+        byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) };
+        random().nextFloat();
+        float expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15];
+        assertEquals(expected, ESVectorUtil.ipFloatBit(q, d), 1e-6);
+    }
+
     public void testBitAndCount() {
         testBasicBitAndImpl(ESVectorUtil::andBitCountLong);
     }

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java

@@ -33,7 +33,7 @@
 public class InternalAdjacencyMatrix extends InternalMultiBucketAggregation<InternalAdjacencyMatrix, InternalAdjacencyMatrix.InternalBucket>
     implements
         AdjacencyMatrix {
-    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements AdjacencyMatrix.Bucket {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucketWritable implements AdjacencyMatrix.Bucket {
 
         private final String key;
         private final long docCount;

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java

@@ -34,7 +34,7 @@ public class InternalTimeSeries extends InternalMultiBucketAggregation<InternalT
     /**
      * A bucket associated with a specific time series (identified by its key)
      */
-    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucketWritable {
         protected long bucketOrd;
         protected final BytesRef key;
         // TODO: make computing docCount optional

modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

@@ -48,7 +48,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
         checker.registerConflictCheck(
             "enabled",
             timestampMapping(true, b -> b.startObject("@timestamp").field("type", "date").endObject()),
-            timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject())
+            timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()),
+            dm -> {}
         );
         checker.registerUpdateCheck(
             timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()),

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml

@@ -3,7 +3,7 @@ setup:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ multi_dense_vector_script_max_sim ]
+          capabilities: [ multi_dense_vector_script_max_sim_with_bugfix ]
       test_runner_features: capabilities
       reason: "Support for multi dense vector max-sim functions capability required"
   - skip:
@@ -136,10 +136,10 @@ setup:
   - match: {hits.total: 2}
 
   - match: {hits.hits.0._id: "1"}
-  - close_to: {hits.hits.0._score: {value: 190, error: 0.01}}
+  - close_to: {hits.hits.0._score: {value: 220, error: 0.01}}
 
   - match: {hits.hits.1._id: "3"}
-  - close_to: {hits.hits.1._score: {value: 125, error: 0.01}}
+  - close_to: {hits.hits.1._score: {value: 147, error: 0.01}}
 ---
 "Test max-sim inv hamming scoring":
   - skip:

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml

@@ -108,7 +108,7 @@ setup:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ byte_float_bit_dot_product ]
+          capabilities: [ byte_float_bit_dot_product_with_bugfix ]
       reason: Capability required to run test
   - do:
       catch: bad_request
@@ -399,7 +399,7 @@ setup:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ byte_float_bit_dot_product ]
+          capabilities: [ byte_float_bit_dot_product_with_bugfix ]
       test_runner_features: [capabilities, close_to]
       reason: Capability required to run test
   - do:
@@ -419,13 +419,13 @@ setup:
   - match: { hits.total: 3 }
 
   - match: {hits.hits.0._id: "2"}
-  - close_to: {hits.hits.0._score: {value: 35.999, error: 0.01}}
+  - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}}
 
   - match: {hits.hits.1._id: "3"}
-  - close_to: {hits.hits.1._score:{value: 27.23, error: 0.01}}
+  - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}}
 
   - match: {hits.hits.2._id: "1"}
-  - close_to: {hits.hits.2._score: {value: 16.57, error: 0.01}}
+  - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}}
 
   - do:
       headers:
@@ -444,20 +444,20 @@ setup:
   - match: { hits.total: 3 }
 
   - match: {hits.hits.0._id: "2"}
-  - close_to: {hits.hits.0._score: {value: 35.999, error: 0.01}}
+  - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}}
 
   - match: {hits.hits.1._id: "3"}
-  - close_to: {hits.hits.1._score:{value: 27.23, error: 0.01}}
+  - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}}
 
   - match: {hits.hits.2._id: "1"}
-  - close_to: {hits.hits.2._score: {value: 16.57, error: 0.01}}
+  - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}}
 ---
 "Dot product with byte":
   - requires:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ byte_float_bit_dot_product ]
+          capabilities: [ byte_float_bit_dot_product_with_bugfix ]
       test_runner_features: capabilities
       reason: Capability required to run test
   - do:
@@ -476,14 +476,14 @@ setup:
 
   - match: { hits.total: 3 }
 
-  - match: {hits.hits.0._id: "1"}
-  - match: {hits.hits.0._score: 248}
+  - match: {hits.hits.0._id: "3"}
+  - match: {hits.hits.0._score: 415}
 
-  - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.1._score: 136}
+  - match: {hits.hits.1._id: "1"}
+  - match: {hits.hits.1._score: 168}
 
-  - match: {hits.hits.2._id: "3"}
-  - match: {hits.hits.2._score: 20}
+  - match: {hits.hits.2._id: "2"}
+  - match: {hits.hits.2._score: 126}
 
   - do:
       headers:
@@ -501,11 +501,11 @@ setup:
 
   - match: { hits.total: 3 }
 
-  - match: {hits.hits.0._id: "1"}
-  - match: {hits.hits.0._score: 248}
+  - match: {hits.hits.0._id: "3"}
+  - match: {hits.hits.0._score: 415}
 
-  - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.1._score: 136}
+  - match: {hits.hits.1._id: "1"}
+  - match: {hits.hits.1._score: 168}
 
-  - match: {hits.hits.2._id: "3"}
-  - match: {hits.hits.2._score: 20}
+  - match: {hits.hits.2._id: "2"}
+  - match: {hits.hits.2._score: 126}

modules/repository-s3/build.gradle

@@ -46,6 +46,7 @@ dependencies {
   yamlRestTestImplementation project(":test:framework")
   yamlRestTestImplementation project(':test:fixtures:s3-fixture')
   yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
+  yamlRestTestImplementation project(':test:fixtures:aws-sts-fixture')
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java

@@ -35,7 +35,14 @@ public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase {
     private static final String BUCKET = "RepositoryS3RestReloadCredentialsIT-bucket-" + HASHED_SEED;
     private static final String BASE_PATH = "RepositoryS3RestReloadCredentialsIT-base-path-" + HASHED_SEED;
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, "ignored");
+    private static volatile String repositoryAccessKey;
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        BUCKET,
+        BASE_PATH,
+        S3HttpFixture.mutableAccessKey(() -> repositoryAccessKey)
+    );
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
 
@@ -68,7 +75,7 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Set up initial credentials
         final var accessKey1 = randomIdentifier();
-        s3Fixture.setAccessKey(accessKey1);
+        repositoryAccessKey = accessKey1;
         keystoreSettings.put("s3.client.default.access_key", accessKey1);
         keystoreSettings.put("s3.client.default.secret_key", randomIdentifier());
         cluster.updateStoredSecureSettings();
@@ -79,14 +86,14 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Rotate credentials in blob store
         final var accessKey2 = randomValueOtherThan(accessKey1, ESTestCase::randomIdentifier);
-        s3Fixture.setAccessKey(accessKey2);
+        repositoryAccessKey = accessKey2;
 
         // Ensure that initial credentials now invalid
         final var accessDeniedException2 = expectThrows(ResponseException.class, () -> client().performRequest(verifyRequest));
         assertThat(accessDeniedException2.getResponse().getStatusLine().getStatusCode(), equalTo(500));
         assertThat(
             accessDeniedException2.getMessage(),
-            allOf(containsString("Bad access key"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
+            allOf(containsString("Access denied"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
         );
 
         // Set up refreshed credentials