diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index dfedfac9d5b04..081d059460653 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -40,8 +40,6 @@ steps: matrix: setup: image: - - windows-2016 - - windows-2019 - windows-2022 agents: provider: gcp diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 8819a5f7f493f..788960c76e150 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -345,8 +345,6 @@ steps: matrix: setup: image: - - windows-2016 - - windows-2019 - windows-2022 agents: provider: gcp diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 86e0623ba5b87..f9f75488f0917 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -38,8 +38,6 @@ steps: matrix: setup: image: - - windows-2016 - - windows-2019 - windows-2022 GRADLE_TASK: - checkPart1 diff --git a/.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml b/.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml index bcf38f51f4a66..6e8fec65f75a5 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml @@ -12,7 +12,7 @@ steps: matrix: setup: image: - - windows-2019 + - windows-2022 PACKAGING_TASK: - default-windows-archive agents: diff --git a/.buildkite/pipelines/pull-request/packaging-tests-windows.yml b/.buildkite/pipelines/pull-request/packaging-tests-windows.yml index 651a82982460f..df5505bea9a62 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-windows.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-windows.yml @@ -10,8 +10,6 @@ steps: matrix: setup: image: - - windows-2016 - - windows-2019 - windows-2022 PACKAGING_TASK: - default-windows-archive diff --git a/muted-tests.yml b/muted-tests.yml index 4fe943df3925b..a404136c9f0ec 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -118,9 +118,6 @@ tests: - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT method: testTracingCrossCluster issue: https://github.com/elastic/elasticsearch/issues/112731 -- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT - method: testInferDeploysDefaultElser - issue: https://github.com/elastic/elasticsearch/issues/114913 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry)} issue: https://github.com/elastic/elasticsearch/issues/115231 @@ -264,9 +261,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=ml/filter_crud/Test update filter} issue: https://github.com/elastic/elasticsearch/issues/116271 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=aggregations/percentiles_hdr_metric/Negative values test} - issue: https://github.com/elastic/elasticsearch/issues/116276 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=ml/get_datafeeds/Test explicit get all datafeeds} issue: https://github.com/elastic/elasticsearch/issues/116284 @@ -285,6 +279,21 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/114723 - class: org.elasticsearch.xpack.search.AsyncSearchSecurityIT issue: https://github.com/elastic/elasticsearch/issues/116293 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test43AutoconfigurationNotTriggeredWhenTlsAlreadyConfigured + issue: https://github.com/elastic/elasticsearch/issues/116317 +- class: org.elasticsearch.xpack.downsample.DownsampleRestIT + issue: https://github.com/elastic/elasticsearch/issues/116326 +- class: org.elasticsearch.xpack.downsample.DownsampleWithBasicRestIT + issue: https://github.com/elastic/elasticsearch/issues/116327 +- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/116328 +- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests + method: testBottomFieldSort + issue: https://github.com/elastic/elasticsearch/issues/116249 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set} + issue: https://github.com/elastic/elasticsearch/issues/116332 # Examples: # diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 23d7af7603d56..f3fd57f3fc8ae 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -61,6 +61,9 @@ excludeList.add('cluster.desired_nodes/20_dry_run/Test validation works for dry // Excluded because they create dot-prefixed indices on older versions excludeList.add('indices.resolve_index/20_resolve_system_index/*') +// Excluded because the error has changed +excludeList.add('aggregations/percentiles_hdr_metric/Negative values test') + BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 4e7577be8e5fc..64fbc6176a31c 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -48,7 +48,6 @@ import static org.junit.Assume.assumeThat; import static org.junit.Assume.assumeTrue; -@PackagingTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/116299") public class ArchiveTests extends PackagingTestCase { @BeforeClass diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCentroid.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCentroid.java index 05dd82fd59c4f..eb789bcdd8a74 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCentroid.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCentroid.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Function; /** * Serialization and merge logic for {@link GeoCentroidAggregator}. @@ -31,24 +30,13 @@ public abstract class InternalCentroid extends InternalAggregation implements CentroidAggregation { protected final SpatialPoint centroid; protected final long count; - private final FieldExtractor firstField; - private final FieldExtractor secondField; - - public InternalCentroid( - String name, - SpatialPoint centroid, - long count, - Map metadata, - FieldExtractor firstField, - FieldExtractor secondField - ) { + + public InternalCentroid(String name, SpatialPoint centroid, long count, Map metadata) { super(name, metadata); assert (centroid == null) == (count == 0); this.centroid = centroid; assert count >= 0; this.count = count; - this.firstField = firstField; - this.secondField = secondField; } protected abstract SpatialPoint centroidFromStream(StreamInput in) throws IOException; @@ -59,7 +47,7 @@ public InternalCentroid( * Read from a stream. */ @SuppressWarnings("this-escape") - protected InternalCentroid(StreamInput in, FieldExtractor firstField, FieldExtractor secondField) throws IOException { + protected InternalCentroid(StreamInput in) throws IOException { super(in); count = in.readVLong(); if (in.readBoolean()) { @@ -67,8 +55,6 @@ protected InternalCentroid(StreamInput in, FieldExtractor firstField, FieldExtra } else { centroid = null; } - this.firstField = firstField; - this.secondField = secondField; } @Override @@ -110,11 +96,11 @@ public void accept(InternalAggregation aggregation) { if (centroidAgg.count > 0) { totalCount += centroidAgg.count; if (Double.isNaN(firstSum)) { - firstSum = centroidAgg.count * firstField.extractor.apply(centroidAgg.centroid); - secondSum = centroidAgg.count * secondField.extractor.apply(centroidAgg.centroid); + firstSum = centroidAgg.count * extractFirst(centroidAgg.centroid); + secondSum = centroidAgg.count * extractSecond(centroidAgg.centroid); } else { - firstSum += centroidAgg.count * firstField.extractor.apply(centroidAgg.centroid); - secondSum += centroidAgg.count * secondField.extractor.apply(centroidAgg.centroid); + firstSum += centroidAgg.count * extractFirst(centroidAgg.centroid); + secondSum += centroidAgg.count * extractSecond(centroidAgg.centroid); } } } @@ -126,6 +112,14 @@ public InternalAggregation get() { }; } + protected abstract String nameFirst(); + + protected abstract double extractFirst(SpatialPoint point); + + protected abstract String nameSecond(); + + protected abstract double extractSecond(SpatialPoint point); + @Override public InternalAggregation finalizeSampling(SamplingContext samplingContext) { return copyWith(centroid, samplingContext.scaleUp(count)); @@ -136,16 +130,6 @@ protected boolean mustReduceOnSingleInternalAgg() { return false; } - protected static class FieldExtractor { - private final String name; - private final Function extractor; - - public FieldExtractor(String name, Function extractor) { - this.name = name; - this.extractor = extractor; - } - } - protected abstract double extractDouble(String name); @Override @@ -174,8 +158,8 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (centroid != null) { builder.startObject(Fields.CENTROID.getPreferredName()); { - builder.field(firstField.name, firstField.extractor.apply(centroid)); - builder.field(secondField.name, secondField.extractor.apply(centroid)); + builder.field(nameFirst(), extractFirst(centroid)); + builder.field(nameSecond(), extractSecond(centroid)); } builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java index 10e301608ec2f..1609046d59708 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.support.SamplingContext; -import org.elasticsearch.xcontent.ParseField; import java.io.IOException; import java.util.Map; @@ -26,21 +25,14 @@ public class InternalGeoCentroid extends InternalCentroid implements GeoCentroid { public InternalGeoCentroid(String name, SpatialPoint centroid, long count, Map metadata) { - super( - name, - centroid, - count, - metadata, - new FieldExtractor("lat", SpatialPoint::getY), - new FieldExtractor("lon", SpatialPoint::getX) - ); + super(name, centroid, count, metadata); } /** * Read from a stream. */ public InternalGeoCentroid(StreamInput in) throws IOException { - super(in, new FieldExtractor("lat", SpatialPoint::getY), new FieldExtractor("lon", SpatialPoint::getX)); + super(in); } public static InternalGeoCentroid empty(String name, Map metadata) { @@ -84,12 +76,27 @@ protected InternalGeoCentroid copyWith(double firstSum, double secondSum, long t } @Override - public InternalAggregation finalizeSampling(SamplingContext samplingContext) { - return new InternalGeoCentroid(name, centroid, samplingContext.scaleUp(count), getMetadata()); + protected String nameFirst() { + return "lat"; + } + + @Override + protected double extractFirst(SpatialPoint point) { + return point.getY(); + } + + @Override + protected String nameSecond() { + return "lon"; + } + + @Override + protected double extractSecond(SpatialPoint point) { + return point.getX(); } - static class Fields { - static final ParseField CENTROID_LAT = new ParseField("lat"); - static final ParseField CENTROID_LON = new ParseField("lon"); + @Override + public InternalAggregation finalizeSampling(SamplingContext samplingContext) { + return new InternalGeoCentroid(name, centroid, samplingContext.scaleUp(count), getMetadata()); } } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java index 3a774a7a37d93..1fef26989d845 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.inference; +import org.elasticsearch.client.Request; import org.elasticsearch.inference.TaskType; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; @@ -27,8 +28,15 @@ public class DefaultEndPointsIT extends InferenceBaseRestTest { private TestThreadPool threadPool; @Before - public void createThreadPool() { + public void setupTest() throws IOException { threadPool = new TestThreadPool(DefaultEndPointsIT.class.getSimpleName()); + + Request loggingSettings = new Request("PUT", "_cluster/settings"); + loggingSettings.setJsonEntity(""" + {"persistent" : { + "logger.org.elasticsearch.xpack.ml.packageloader" : "DEBUG" + }}"""); + client().performRequest(loggingSettings); } @After @@ -64,7 +72,7 @@ private static void assertDefaultElserConfig(Map modelConfig) { assertThat( modelConfig.toString(), adaptiveAllocations, - Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8)) + Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) ); } @@ -99,7 +107,7 @@ private static void assertDefaultE5Config(Map modelConfig) { assertThat( modelConfig.toString(), adaptiveAllocations, - Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8)) + Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) ); } } diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java index e92aff74be463..2e08b845f6593 100644 --- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java +++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ml.packageloader.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; @@ -56,10 +58,12 @@ */ final class ModelLoaderUtils { + private static final Logger logger = LogManager.getLogger(ModelLoaderUtils.class); + public static String METADATA_FILE_EXTENSION = ".metadata.json"; public static String MODEL_FILE_EXTENSION = ".pt"; - private static ByteSizeValue VOCABULARY_SIZE_LIMIT = new ByteSizeValue(20, ByteSizeUnit.MB); + private static final ByteSizeValue VOCABULARY_SIZE_LIMIT = new ByteSizeValue(20, ByteSizeUnit.MB); private static final String VOCABULARY = "vocabulary"; private static final String MERGES = "merges"; private static final String SCORES = "scores"; @@ -83,6 +87,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {} private final AtomicInteger currentPart; private final int lastPartNumber; private final byte[] buf; + private final RequestRange range; // TODO debug only HttpStreamChunker(URI uri, RequestRange range, int chunkSize) { var inputStream = getHttpOrHttpsInputStream(uri, range); @@ -91,6 +96,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {} this.lastPartNumber = range.startPart() + range.numParts(); this.currentPart = new AtomicInteger(range.startPart()); this.buf = new byte[chunkSize]; + this.range = range; } // This ctor exists for testing purposes only. @@ -100,6 +106,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {} this.lastPartNumber = range.startPart() + range.numParts(); this.currentPart = new AtomicInteger(range.startPart()); this.buf = new byte[chunkSize]; + this.range = range; } public boolean hasNext() { @@ -113,6 +120,7 @@ public BytesAndPartIndex next() throws IOException { int read = inputStream.read(buf, bytesRead, chunkSize - bytesRead); // EOF?? if (read == -1) { + logger.debug("end of stream, " + bytesRead + " bytes read"); break; } bytesRead += read; @@ -122,6 +130,7 @@ public BytesAndPartIndex next() throws IOException { totalBytesRead.addAndGet(bytesRead); return new BytesAndPartIndex(new BytesArray(buf, 0, bytesRead), currentPart.getAndIncrement()); } else { + logger.warn("Empty part in range " + range + ", current part=" + currentPart.get() + ", last part=" + lastPartNumber); return new BytesAndPartIndex(BytesArray.EMPTY, currentPart.get()); } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/InternalCartesianCentroid.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/InternalCartesianCentroid.java index e009e07d35aa4..63f43458c79b5 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/InternalCartesianCentroid.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/InternalCartesianCentroid.java @@ -13,7 +13,6 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.metrics.InternalCentroid; import org.elasticsearch.search.aggregations.support.SamplingContext; -import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.spatial.common.CartesianPoint; import java.io.IOException; @@ -25,14 +24,14 @@ public class InternalCartesianCentroid extends InternalCentroid implements CartesianCentroid { public InternalCartesianCentroid(String name, SpatialPoint centroid, long count, Map metadata) { - super(name, centroid, count, metadata, new FieldExtractor("x", SpatialPoint::getX), new FieldExtractor("y", SpatialPoint::getY)); + super(name, centroid, count, metadata); } /** * Read from a stream. */ public InternalCartesianCentroid(StreamInput in) throws IOException { - super(in, new FieldExtractor("x", SpatialPoint::getX), new FieldExtractor("y", SpatialPoint::getY)); + super(in); } @Override @@ -80,8 +79,23 @@ public InternalAggregation finalizeSampling(SamplingContext samplingContext) { return new InternalCartesianCentroid(name, centroid, samplingContext.scaleUp(count), getMetadata()); } - static class Fields { - static final ParseField CENTROID_X = new ParseField("x"); - static final ParseField CENTROID_Y = new ParseField("y"); + @Override + protected String nameFirst() { + return "x"; + } + + @Override + protected double extractFirst(SpatialPoint point) { + return point.getX(); + } + + @Override + protected String nameSecond() { + return "y"; + } + + @Override + protected double extractSecond(SpatialPoint point) { + return point.getY(); } }