Merge branch 'main' into internalAggregationName

.buildkite/pipelines/periodic-packaging.template.yml

@@ -40,8 +40,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
         agents:
           provider: gcp

.buildkite/pipelines/periodic-packaging.yml

@@ -345,8 +345,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
         agents:
           provider: gcp

.buildkite/pipelines/periodic-platform-support.yml

@@ -38,8 +38,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
             GRADLE_TASK:
               - checkPart1

.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml

@@ -12,7 +12,7 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2019
+              - windows-2022
             PACKAGING_TASK:
               - default-windows-archive
         agents:

.buildkite/pipelines/pull-request/packaging-tests-windows.yml

@@ -10,8 +10,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
             PACKAGING_TASK:
               - default-windows-archive

muted-tests.yml

@@ -118,9 +118,6 @@ tests:
 - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT
   method: testTracingCrossCluster
   issue: https://github.com/elastic/elasticsearch/issues/112731
-- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
-  method: testInferDeploysDefaultElser
-  issue: https://github.com/elastic/elasticsearch/issues/114913
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry)}
   issue: https://github.com/elastic/elasticsearch/issues/115231
@@ -264,9 +261,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/filter_crud/Test update filter}
   issue: https://github.com/elastic/elasticsearch/issues/116271
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=aggregations/percentiles_hdr_metric/Negative values test}
-  issue: https://github.com/elastic/elasticsearch/issues/116276
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/get_datafeeds/Test explicit get all datafeeds}
   issue: https://github.com/elastic/elasticsearch/issues/116284
@@ -285,6 +279,21 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/114723
 - class: org.elasticsearch.xpack.search.AsyncSearchSecurityIT
   issue: https://github.com/elastic/elasticsearch/issues/116293
+- class: org.elasticsearch.packaging.test.ArchiveTests
+  method: test43AutoconfigurationNotTriggeredWhenTlsAlreadyConfigured
+  issue: https://github.com/elastic/elasticsearch/issues/116317
+- class: org.elasticsearch.xpack.downsample.DownsampleRestIT
+  issue: https://github.com/elastic/elasticsearch/issues/116326
+- class: org.elasticsearch.xpack.downsample.DownsampleWithBasicRestIT
+  issue: https://github.com/elastic/elasticsearch/issues/116327
+- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/116328
+- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
+  method: testBottomFieldSort
+  issue: https://github.com/elastic/elasticsearch/issues/116249
+- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
+  method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set}
+  issue: https://github.com/elastic/elasticsearch/issues/116332
 
 # Examples:
 #

qa/mixed-cluster/build.gradle

@@ -61,6 +61,9 @@ excludeList.add('cluster.desired_nodes/20_dry_run/Test validation works for dry
 // Excluded because they create dot-prefixed indices on older versions
 excludeList.add('indices.resolve_index/20_resolve_system_index/*')
 
+// Excluded because the error has changed
+excludeList.add('aggregations/percentiles_hdr_metric/Negative values test')
+
 BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
 
   if (bwcVersion != VersionProperties.getElasticsearchVersion()) {

qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -48,7 +48,6 @@
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
 
-@PackagingTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/116299")
 public class ArchiveTests extends PackagingTestCase {
 
     @BeforeClass

server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCentroid.java

@@ -23,32 +23,20 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Function;
 
 /**
  * Serialization and merge logic for {@link GeoCentroidAggregator}.
  */
 public abstract class InternalCentroid extends InternalAggregation implements CentroidAggregation {
     protected final SpatialPoint centroid;
     protected final long count;
-    private final FieldExtractor firstField;
-    private final FieldExtractor secondField;
-
-    public InternalCentroid(
-        String name,
-        SpatialPoint centroid,
-        long count,
-        Map<String, Object> metadata,
-        FieldExtractor firstField,
-        FieldExtractor secondField
-    ) {
+
+    public InternalCentroid(String name, SpatialPoint centroid, long count, Map<String, Object> metadata) {
         super(name, metadata);
         assert (centroid == null) == (count == 0);
         this.centroid = centroid;
         assert count >= 0;
         this.count = count;
-        this.firstField = firstField;
-        this.secondField = secondField;
     }
 
     protected abstract SpatialPoint centroidFromStream(StreamInput in) throws IOException;
@@ -59,16 +47,14 @@ public InternalCentroid(
      * Read from a stream.
      */
     @SuppressWarnings("this-escape")
-    protected InternalCentroid(StreamInput in, FieldExtractor firstField, FieldExtractor secondField) throws IOException {
+    protected InternalCentroid(StreamInput in) throws IOException {
         super(in);
         count = in.readVLong();
         if (in.readBoolean()) {
             centroid = centroidFromStream(in);
         } else {
             centroid = null;
         }
-        this.firstField = firstField;
-        this.secondField = secondField;
     }
 
     @Override
@@ -110,11 +96,11 @@ public void accept(InternalAggregation aggregation) {
                 if (centroidAgg.count > 0) {
                     totalCount += centroidAgg.count;
                     if (Double.isNaN(firstSum)) {
-                        firstSum = centroidAgg.count * firstField.extractor.apply(centroidAgg.centroid);
-                        secondSum = centroidAgg.count * secondField.extractor.apply(centroidAgg.centroid);
+                        firstSum = centroidAgg.count * extractFirst(centroidAgg.centroid);
+                        secondSum = centroidAgg.count * extractSecond(centroidAgg.centroid);
                     } else {
-                        firstSum += centroidAgg.count * firstField.extractor.apply(centroidAgg.centroid);
-                        secondSum += centroidAgg.count * secondField.extractor.apply(centroidAgg.centroid);
+                        firstSum += centroidAgg.count * extractFirst(centroidAgg.centroid);
+                        secondSum += centroidAgg.count * extractSecond(centroidAgg.centroid);
                     }
                 }
             }
@@ -126,6 +112,14 @@ public InternalAggregation get() {
         };
     }
 
+    protected abstract String nameFirst();
+
+    protected abstract double extractFirst(SpatialPoint point);
+
+    protected abstract String nameSecond();
+
+    protected abstract double extractSecond(SpatialPoint point);
+
     @Override
     public InternalAggregation finalizeSampling(SamplingContext samplingContext) {
         return copyWith(centroid, samplingContext.scaleUp(count));
@@ -136,16 +130,6 @@ protected boolean mustReduceOnSingleInternalAgg() {
         return false;
     }
 
-    protected static class FieldExtractor {
-        private final String name;
-        private final Function<SpatialPoint, Double> extractor;
-
-        public FieldExtractor(String name, Function<SpatialPoint, Double> extractor) {
-            this.name = name;
-            this.extractor = extractor;
-        }
-    }
-
     protected abstract double extractDouble(String name);
 
     @Override
@@ -174,8 +158,8 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         if (centroid != null) {
             builder.startObject(Fields.CENTROID.getPreferredName());
             {
-                builder.field(firstField.name, firstField.extractor.apply(centroid));
-                builder.field(secondField.name, secondField.extractor.apply(centroid));
+                builder.field(nameFirst(), extractFirst(centroid));
+                builder.field(nameSecond(), extractSecond(centroid));
             }
             builder.endObject();
         }

server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.support.SamplingContext;
-import org.elasticsearch.xcontent.ParseField;
 
 import java.io.IOException;
 import java.util.Map;
@@ -26,21 +25,14 @@
 public class InternalGeoCentroid extends InternalCentroid implements GeoCentroid {
 
     public InternalGeoCentroid(String name, SpatialPoint centroid, long count, Map<String, Object> metadata) {
-        super(
-            name,
-            centroid,
-            count,
-            metadata,
-            new FieldExtractor("lat", SpatialPoint::getY),
-            new FieldExtractor("lon", SpatialPoint::getX)
-        );
+        super(name, centroid, count, metadata);
     }
 
     /**
      * Read from a stream.
      */
     public InternalGeoCentroid(StreamInput in) throws IOException {
-        super(in, new FieldExtractor("lat", SpatialPoint::getY), new FieldExtractor("lon", SpatialPoint::getX));
+        super(in);
     }
 
     public static InternalGeoCentroid empty(String name, Map<String, Object> metadata) {
@@ -84,12 +76,27 @@ protected InternalGeoCentroid copyWith(double firstSum, double secondSum, long t
     }
 
     @Override
-    public InternalAggregation finalizeSampling(SamplingContext samplingContext) {
-        return new InternalGeoCentroid(name, centroid, samplingContext.scaleUp(count), getMetadata());
+    protected String nameFirst() {
+        return "lat";
+    }
+
+    @Override
+    protected double extractFirst(SpatialPoint point) {
+        return point.getY();
+    }
+
+    @Override
+    protected String nameSecond() {
+        return "lon";
+    }
+
+    @Override
+    protected double extractSecond(SpatialPoint point) {
+        return point.getX();
     }
 
-    static class Fields {
-        static final ParseField CENTROID_LAT = new ParseField("lat");
-        static final ParseField CENTROID_LON = new ParseField("lon");
+    @Override
+    public InternalAggregation finalizeSampling(SamplingContext samplingContext) {
+        return new InternalGeoCentroid(name, centroid, samplingContext.scaleUp(count), getMetadata());
     }
 }

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.inference;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService;
@@ -27,8 +28,15 @@ public class DefaultEndPointsIT extends InferenceBaseRestTest {
     private TestThreadPool threadPool;
 
     @Before
-    public void createThreadPool() {
+    public void setupTest() throws IOException {
         threadPool = new TestThreadPool(DefaultEndPointsIT.class.getSimpleName());
+
+        Request loggingSettings = new Request("PUT", "_cluster/settings");
+        loggingSettings.setJsonEntity("""
+            {"persistent" : {
+                    "logger.org.elasticsearch.xpack.ml.packageloader" : "DEBUG"
+                }}""");
+        client().performRequest(loggingSettings);
     }
 
     @After
@@ -64,7 +72,7 @@ private static void assertDefaultElserConfig(Map<String, Object> modelConfig) {
         assertThat(
             modelConfig.toString(),
             adaptiveAllocations,
-            Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8))
+            Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32))
         );
     }
 
@@ -99,7 +107,7 @@ private static void assertDefaultE5Config(Map<String, Object> modelConfig) {
         assertThat(
             modelConfig.toString(),
             adaptiveAllocations,
-            Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8))
+            Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32))
         );
     }
 }

x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.ml.packageloader.action;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
@@ -56,10 +58,12 @@
  */
 final class ModelLoaderUtils {
 
+    private static final Logger logger = LogManager.getLogger(ModelLoaderUtils.class);
+
     public static String METADATA_FILE_EXTENSION = ".metadata.json";
     public static String MODEL_FILE_EXTENSION = ".pt";
 
-    private static ByteSizeValue VOCABULARY_SIZE_LIMIT = new ByteSizeValue(20, ByteSizeUnit.MB);
+    private static final ByteSizeValue VOCABULARY_SIZE_LIMIT = new ByteSizeValue(20, ByteSizeUnit.MB);
     private static final String VOCABULARY = "vocabulary";
     private static final String MERGES = "merges";
     private static final String SCORES = "scores";
@@ -83,6 +87,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {}
         private final AtomicInteger currentPart;
         private final int lastPartNumber;
         private final byte[] buf;
+        private final RequestRange range;  // TODO debug only
 
         HttpStreamChunker(URI uri, RequestRange range, int chunkSize) {
             var inputStream = getHttpOrHttpsInputStream(uri, range);
@@ -91,6 +96,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {}
             this.lastPartNumber = range.startPart() + range.numParts();
             this.currentPart = new AtomicInteger(range.startPart());
             this.buf = new byte[chunkSize];
+            this.range = range;
         }
 
         // This ctor exists for testing purposes only.
@@ -100,6 +106,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {}
             this.lastPartNumber = range.startPart() + range.numParts();
             this.currentPart = new AtomicInteger(range.startPart());
             this.buf = new byte[chunkSize];
+            this.range = range;
         }
 
         public boolean hasNext() {
@@ -113,6 +120,7 @@ public BytesAndPartIndex next() throws IOException {
                 int read = inputStream.read(buf, bytesRead, chunkSize - bytesRead);
                 // EOF??
                 if (read == -1) {
+                    logger.debug("end of stream, " + bytesRead + " bytes read");
                     break;
                 }
                 bytesRead += read;
@@ -122,6 +130,7 @@ public BytesAndPartIndex next() throws IOException {
                 totalBytesRead.addAndGet(bytesRead);
                 return new BytesAndPartIndex(new BytesArray(buf, 0, bytesRead), currentPart.getAndIncrement());
             } else {
+                logger.warn("Empty part in range " + range + ", current part=" + currentPart.get() + ", last part=" + lastPartNumber);
                 return new BytesAndPartIndex(BytesArray.EMPTY, currentPart.get());
             }
         }