Merge remote-tracking branch 'es/main' into esql_synthetic_source_loa…

…d_ignored_source

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -272,8 +272,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.3
+      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -286,7 +286,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

.buildkite/pipelines/periodic.yml

@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.3#bwcTest
+      - label: 8.15.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -429,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -471,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -14,7 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"

docs/changelog/112881.yaml

@@ -0,0 +1,5 @@
+pr: 112881
+summary: "ESQL: Remove parent from `FieldAttribute`"
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/connector/docs/connectors-content-extraction.asciidoc

@@ -90,7 +90,7 @@ include::_connectors-list-local-content-extraction.asciidoc[]
 Self-hosted content extraction is handled by a *separate* extraction service.
 
 The versions for the extraction service do not align with the Elastic stack.
-For version `8.11.x`, you should use extraction service version `0.3.x`.
+For versions after `8.11.x` (including {version}), you should use extraction service version `0.3.x`.
 
 You can run the service with the following command:
 

muted-tests.yml

@@ -343,6 +343,15 @@ tests:
 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
   method: testInferDeploysDefaultElser
   issue: https://github.com/elastic/elasticsearch/issues/114913
+- class: org.elasticsearch.upgrades.MultiVersionRepositoryAccessIT
+  method: testUpgradeMovesRepoToNewMetaVersion
+  issue: https://github.com/elastic/elasticsearch/issues/114994
+- class: org.elasticsearch.upgrades.MultiVersionRepositoryAccessIT
+  method: testReadOnlyRepo
+  issue: https://github.com/elastic/elasticsearch/issues/114997
+- class: org.elasticsearch.upgrades.MultiVersionRepositoryAccessIT
+  method: testCreateAndRestoreSnapshot
+  issue: https://github.com/elastic/elasticsearch/issues/114998
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -175,6 +175,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT = def(8_771_00_0);
     public static final TransportVersion CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY = def(8_772_00_0);
     public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0);
+    public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_00_0);
+    public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/Version.java

@@ -186,6 +186,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_15_1 = new Version(8_15_01_99);
     public static final Version V_8_15_2 = new Version(8_15_02_99);
     public static final Version V_8_15_3 = new Version(8_15_03_99);
+    public static final Version V_8_15_4 = new Version(8_15_04_99);
     public static final Version V_8_16_0 = new Version(8_16_00_99);
     public static final Version V_8_17_0 = new Version(8_17_00_99);
     public static final Version V_9_0_0 = new Version(9_00_00_99);

server/src/main/java/org/elasticsearch/action/search/SearchRequest.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.Rewriteable;
@@ -254,8 +255,10 @@ public SearchRequest(StreamInput in) throws IOException {
             finalReduce = true;
         }
         ccsMinimizeRoundtrips = in.readBoolean();
-        if (in.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE) && in.readBoolean()) {
-            Version.readVersion(in); // and drop on the floor
+        if ((in.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)
+            || in.getTransportVersion().onOrAfter(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE)) && in.readBoolean()) {
+            @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)  // this can be removed (again) when the v9 transport version can diverge
+            Version v = Version.readVersion(in); // and drop on the floor
         }
         waitForCheckpoints = in.readMap(StreamInput::readLongArray);
         waitForCheckpointsTimeout = in.readTimeValue();
@@ -291,7 +294,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(finalReduce);
         }
         out.writeBoolean(ccsMinimizeRoundtrips);
-        if (out.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)) {
+        if (out.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)
+            || out.getTransportVersion().onOrAfter(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE)) {
             out.writeBoolean(false);
         }
         out.writeMap(waitForCheckpoints, StreamOutput::writeLongArray);

server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java

@@ -21,8 +21,6 @@
 import java.io.IOException;
 import java.util.Locale;
 
-import static org.elasticsearch.common.geo.GeoUtils.normalizeLat;
-import static org.elasticsearch.common.geo.GeoUtils.normalizeLon;
 import static org.elasticsearch.common.geo.GeoUtils.quantizeLat;
 
 /**
@@ -113,15 +111,13 @@ public static int checkPrecisionRange(int precision) {
      * Calculates the x-coordinate in the tile grid for the specified longitude given
      * the number of tile columns for a pre-determined zoom-level.
      *
-     * @param longitude the longitude to use when determining the tile x-coordinate
+     * @param longitude the longitude to use when determining the tile x-coordinate. Longitude is in degrees
+     *                  and must be between -180 and 180 degrees.
      * @param tiles     the number of tiles per row for a pre-determined zoom-level
      */
     public static int getXTile(double longitude, int tiles) {
-        // normalizeLon treats this as 180, which is not friendly for tile mapping
-        if (longitude == -180) {
-            return 0;
-        }
-        final double xTile = (normalizeLon(longitude) + 180.0) / 360.0 * tiles;
+        assert longitude >= -180 && longitude <= 180 : "Longitude must be between -180 and 180 degrees";
+        final double xTile = (longitude + 180.0) / 360.0 * tiles;
         // Edge values may generate invalid values, and need to be clipped.
         return Math.max(0, Math.min(tiles - 1, (int) Math.floor(xTile)));
     }
@@ -130,11 +126,13 @@ public static int getXTile(double longitude, int tiles) {
      * Calculates the y-coordinate in the tile grid for the specified longitude given
      * the number of tile rows for pre-determined zoom-level.
      *
-     * @param latitude  the latitude to use when determining the tile y-coordinate
+     * @param latitude  the latitude to use when determining the tile y-coordinate. Latitude is in degrees
+     *                  and must be between -90 and 90 degrees.
      * @param tiles     the number of tiles per column for a pre-determined zoom-level
      */
     public static int getYTile(double latitude, int tiles) {
-        final double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(normalizeLat(latitude)));
+        assert latitude >= -90 && latitude <= 90 : "Latitude must be between -90 and 90 degrees";
+        final double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(latitude));
         final double yTile = (0.5 - (ESSloppyMath.log((1.0 + latSin) / (1.0 - latSin)) / PI_TIMES_4)) * tiles;
         // Edge values may generate invalid values, and need to be clipped.
         // For example, polar regions (above/below lat 85.05112878) get normalized.

server/src/main/resources/org/elasticsearch/TransportVersions.csv

@@ -130,3 +130,4 @@
 8.15.0,8702002
 8.15.1,8702002
 8.15.2,8702003
+8.15.3,8702003

server/src/main/resources/org/elasticsearch/index/IndexVersions.csv

@@ -130,3 +130,4 @@
 8.15.0,8512000
 8.15.1,8512000
 8.15.2,8512000
+8.15.3,8512000

server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java

@@ -16,9 +16,12 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ArrayUtils;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
@@ -102,6 +105,23 @@ public void testSerialization() throws Exception {
         assertNotSame(deserializedRequest, searchRequest);
     }
 
+    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)  // this can be removed when the affected transport version constants are collapsed
+    public void testSerializationConstants() throws Exception {
+        SearchRequest searchRequest = createSearchRequest();
+
+        // something serialized with previous version to remove, should read correctly with the reversion
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setTransportVersion(TransportVersionUtils.getPreviousVersion(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE));
+            searchRequest.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                in.setTransportVersion(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE);
+                SearchRequest copiedRequest = new SearchRequest(in);
+                assertEquals(copiedRequest, searchRequest);
+                assertEquals(copiedRequest.hashCode(), searchRequest.hashCode());
+            }
+        }
+    }
+
     public void testSerializationMultiKNN() throws Exception {
         SearchRequest searchRequest = createSearchRequest();
         if (searchRequest.source() == null) {

server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java

@@ -11,6 +11,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -173,8 +174,8 @@ public void testGeoTileFormat() {
         assertEquals("29/536869420/0", DocValueFormat.GEOTILE.format(longEncode(179.999, 89.999, 29)));
         assertEquals("29/1491/536870911", DocValueFormat.GEOTILE.format(longEncode(-179.999, -89.999, 29)));
         assertEquals("2/2/1", DocValueFormat.GEOTILE.format(longEncode(1, 1, 2)));
-        assertEquals("1/1/0", DocValueFormat.GEOTILE.format(longEncode(13, 95, 1)));
-        assertEquals("1/1/1", DocValueFormat.GEOTILE.format(longEncode(13, -95, 1)));
+        assertEquals("1/1/0", DocValueFormat.GEOTILE.format(longEncode(13, GeoUtils.normalizeLat(95), 1)));
+        assertEquals("1/1/1", DocValueFormat.GEOTILE.format(longEncode(13, GeoUtils.normalizeLat(-95), 1)));
     }
 
     public void testRawParse() {