Merge branch 'main' into test/die-with-dignity-pid5

.ci/packer_cache.ps1

@@ -0,0 +1,16 @@
+$RUNBLD_DIR = 'C:\Program Files\runbld\src\runbld-7.0.3'
+$RUNBLD = Join-Path $RUNBLD_DIR 'runbld'
+
+# Check if 7.0.3 doesn't already exist
+if (-not (Test-Path $RUNBLD)) {
+  New-Item -Path $RUNBLD_DIR -ItemType Directory -Force
+  Invoke-WebRequest -Uri 'http://packages.elasticsearch.org.s3.amazonaws.com/infra/runbld-7.0.3' -OutFile $RUNBLD
+
+  $RUNBLD_HARDLINK_DIR = 'C:\Program Files\infra\bin'
+  $RUNBLD_HARDLINK = Join-Path $RUNBLD_HARDLINK_DIR 'runbld'
+
+  Remove-Item -Path $RUNBLD_HARDLINK -Force
+  New-Item -Path $RUNBLD_HARDLINK_DIR -ItemType Directory -Force
+
+  fsutil hardlink create $RUNBLD_HARDLINK $RUNBLD
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -18,10 +18,10 @@
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
-import org.elasticsearch.xpack.esql.planner.EvalMapper;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
@@ -94,7 +94,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     new EsField("timestamp", DataTypes.DATETIME, Map.of(), true)
                 );
                 yield EvalMapper.toEvaluator(
-                    new DateTrunc(Source.EMPTY, timestamp, new Literal(Source.EMPTY, Duration.ofHours(24), EsqlDataTypes.TIME_DURATION)),
+                    new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), EsqlDataTypes.TIME_DURATION), timestamp),
                     layout(timestamp)
                 ).get();
             }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -196,7 +196,7 @@ public void benchmark() {
                 case "double" -> {
                     DoubleVector values = op.getOutput().<DoubleBlock>getBlock(1).asVector();
                     for (int p = 0; p < values.getPositionCount(); p++) {
-                        sum += values.getDouble(p);
+                        sum += (long) values.getDouble(p);
                     }
                 }
                 case "keyword" -> {

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -10,13 +10,13 @@
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
@@ -44,7 +44,7 @@ public static MapperService create(String mappings) {
         Settings settings = Settings.builder()
             .put("index.number_of_replicas", 0)
             .put("index.number_of_shards", 1)
-            .put("index.version.created", Version.CURRENT)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put("index.mapping.total_fields.limit", 10000)
             .build();
         IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.benchmark.routing.allocation;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -19,6 +18,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexVersion;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -128,7 +128,7 @@ public void setUp() throws Exception {
         for (int i = 1; i <= numIndices; i++) {
             mb.put(
                 IndexMetadata.builder("test_" + i)
-                    .settings(Settings.builder().put("index.version.created", Version.CURRENT))
+                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
                     .numberOfShards(numShards)
                     .numberOfReplicas(numReplicas)
             );

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.benchmark.routing.allocation;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -28,6 +27,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.health.HealthIndicatorResult;
 import org.elasticsearch.health.node.HealthInfo;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.TaskManager;
@@ -121,7 +121,7 @@ public void setUp() throws Exception {
             IndexMetadata indexMetadata = IndexMetadata.builder("test_" + i)
                 .settings(
                     Settings.builder()
-                        .put("index.version.created", Version.CURRENT)
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(DataTier.TIER_PREFERENCE_SETTING.getKey(), "data_warm")
                         .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm")
                 )

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -17,18 +17,17 @@
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
@@ -140,7 +139,6 @@ protected SearchExecutionContext buildSearchExecutionContext() {
             0,
             0,
             mapperService.getIndexSettings(),
-            ClusterSettings.createBuiltInClusterSettings(),
             null,
             (ft, fdc) -> ft.fielddataBuilder(fdc).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()),
             mapperService,
@@ -165,7 +163,7 @@ protected final MapperService createMapperService(String mappings) {
         Settings settings = Settings.builder()
             .put("index.number_of_replicas", 0)
             .put("index.number_of_shards", 1)
-            .put("index.version.created", Version.CURRENT)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .build();
         IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(meta, settings);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java

@@ -8,13 +8,17 @@
 
 package org.elasticsearch.gradle.internal;
 
+import org.elasticsearch.gradle.VersionProperties;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.TaskAction;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 
 import java.util.Collection;
 import java.util.stream.Collectors;
@@ -24,7 +28,12 @@
 import static org.elasticsearch.gradle.DistributionDownloadPlugin.DISTRO_EXTRACTED_CONFIG_PREFIX;
 import static org.elasticsearch.gradle.internal.test.rest.compat.compat.LegacyYamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME;
 
-public class ResolveAllDependencies extends DefaultTask {
+public abstract class ResolveAllDependencies extends DefaultTask {
+
+    private boolean resolveJavaToolChain = false;
+
+    @Inject
+    protected abstract JavaToolchainService getJavaToolchainService();
 
     private final ObjectFactory objectFactory;
 
@@ -43,7 +52,19 @@ public FileCollection getResolvedArtifacts() {
 
     @TaskAction
     void resolveAll() {
-        // do nothing, dependencies are resolved when snapshotting task inputs
+        if (resolveJavaToolChain) {
+            resolveDefaultJavaToolChain();
+        }
+    }
+
+    private void resolveDefaultJavaToolChain() {
+        getJavaToolchainService().launcherFor(javaToolchainSpec -> {
+            String bundledVendor = VersionProperties.getBundledJdkVendor();
+            String bundledJdkMajorVersion = VersionProperties.getBundledJdkMajorVersion();
+            javaToolchainSpec.getLanguageVersion().set(JavaLanguageVersion.of(bundledJdkMajorVersion));
+            javaToolchainSpec.getVendor()
+                .set(bundledVendor.equals("openjdk") ? JvmVendorSpec.ORACLE : JvmVendorSpec.matching(bundledVendor));
+        }).get();
     }
 
     @Internal
@@ -55,6 +76,15 @@ public void setConfigs(Collection<Configuration> configs) {
         this.configs = configs;
     }
 
+    @Internal
+    public boolean isResolveJavaToolChain() {
+        return resolveJavaToolChain;
+    }
+
+    public void setResolveJavaToolChain(boolean resolveJavaToolChain) {
+        this.resolveJavaToolChain = resolveJavaToolChain;
+    }
+
     private static boolean canBeResolved(Configuration configuration) {
         if (configuration.isCanBeResolved() == false) {
             return false;
@@ -67,4 +97,5 @@ private static boolean canBeResolved(Configuration configuration) {
         return configuration.getName().startsWith(DISTRO_EXTRACTED_CONFIG_PREFIX) == false
             && configuration.getName().equals(BWC_MINOR_CONFIG_NAME) == false;
     }
+
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java

@@ -11,6 +11,9 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -28,6 +31,8 @@
  * </ul>
  */
 public class ChangelogEntry {
+    private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class);
+
     private Integer pr;
     private List<Integer> issues;
     private String area;
@@ -48,6 +53,7 @@ public static ChangelogEntry parse(File file) {
         try {
             return yamlMapper.readValue(file, ChangelogEntry.class);
         } catch (IOException e) {
+            LOGGER.error("Failed to parse changelog from " + file.getAbsolutePath(), e);
             throw new UncheckedIOException(e);
         }
     }

build-tools-internal/version.properties

@@ -26,7 +26,7 @@ antlr4            = 4.11.1
 #  - x-pack/plugin/security
 bouncycastle=1.64
 # used by security and idp (need to be in sync due to cross-dependency in testing)
-opensaml = 4.0.1
+opensaml = 4.3.0
 
 # client dependencies
 httpclient        = 4.5.13

build.gradle

@@ -200,6 +200,7 @@ allprojects {
 
   tasks.register('resolveAllDependencies', ResolveAllDependencies) {
     configs = project.configurations
+    resolveJavaToolChain = true
     if (project.path.contains("fixture")) {
       dependsOn tasks.withType(ComposePull)
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java

@@ -219,7 +219,8 @@ private static Process createProcess(
         command.add(esHome.resolve("lib").toString());
         // Special circumstances require some modules (not depended on by the main server module) to be explicitly added:
         command.add("--add-modules=jdk.net"); // needed to reflectively set extended socket options
-        command.add("--add-modules=org.elasticsearch.preallocate"); // needed on boot layer as target to open java.io
+        // we control the module path, which may have additional modules not required by server
+        command.add("--add-modules=ALL-MODULE-PATH");
         command.add("-m");
         command.add("org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch");
 

docs/build.gradle

@@ -100,7 +100,6 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-  requiresFeature 'es.dlm_feature_flag_enabled', Version.fromString("8.8.0")
 
   // build the cluster with all plugins
   project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
@@ -122,9 +121,6 @@ tasks.named("yamlRestTest").configure {
   doFirst {
     delete("${buildDir}/cluster/shared/repo")
   }
-  if (BuildParams.isSnapshotBuild() == false) {
-    systemProperty 'es.dlm_feature_flag_enabled', 'true'
-  }
 }
 
 tasks.named("forbiddenPatterns").configure {

docs/changelog/97319.yaml

@@ -2,7 +2,19 @@ pr: 97319
 summary: Change pre-configured and cached analyzer components to use IndexVersion instead of Version
 area: "Analysis"
 type: breaking-java
-breaking: "Java API"
+breaking:
+  title: Change pre-configured and cached analyzer components to use IndexVersion instead of Version
+  area: Java API
+  details: |-
+    This PR changes the types used to obtain pre-configured components from Version to IndexVersion,
+    with corresponding changes to method names.
+
+    Prior to 8.10, there is a one-to-one mapping between node version and index version, with corresponding constants
+    in the IndexVersion class.
+    Starting in 8.10, IndexVersion is versioned independently of node version, and will be a simple incrementing number.
+    For more information on how to use IndexVersion and other version types, please see the contributing guide.
+  impact: Analysis components now take IndexVersion instead of Version
+  notable: false
 issues: []
 highlight:
   title: Change pre-configured and cached analyzer components to use IndexVersion instead of Version

docs/changelog/97397.yaml

@@ -0,0 +1,5 @@
+pr: 97397
+summary: Return a 410 (Gone) status code for unavailable API endpoints
+area: Infra/REST API
+type: enhancement
+issues: []

docs/changelog/97409.yaml

@@ -0,0 +1,5 @@
+pr: 97409
+summary: Trim stored fields for `_id` field in tsdb
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/97972.yaml

@@ -0,0 +1,6 @@
+pr: 97972
+summary: Automatically flatten objects when subobjects:false
+area: Mapping
+type: enhancement
+issues:
+ - 88934

docs/changelog/98204.yaml

@@ -1,5 +1,13 @@
 pr: 98204
 summary: Introduce executor for concurrent search
 area: Search
-type: enhancement
+type: feature
 issues: []
+highlight:
+  title: Enable parallel knn search across segments
+  body: |-
+    Elasticsearch has until now performed search sequentially across the
+    segments within each shard. This change makes knn queries faster on shards
+    that are made of more than one segment, by rewriting and collecting each
+    segment in parallel.
+  notable: true

docs/changelog/98265.yaml

@@ -0,0 +1,5 @@
+pr: 98265
+summary: Do not assign ignored shards
+area: Allocation
+type: bug
+issues: []

docs/changelog/98332.yaml

@@ -0,0 +1,6 @@
+pr: 98332
+summary: Correct behaviour of `ContentPath::remove()`
+area: Mapping
+type: bug
+issues:
+ - 98327

docs/changelog/98337.yaml

@@ -0,0 +1,5 @@
+pr: 98337
+summary: TopN sorting with min and max for multi-value fields
+area: ES|QL
+type: feature
+issues: []

docs/changelog/98550.yaml

@@ -0,0 +1,5 @@
+pr: 98550
+summary: Report a node's "roles" setting in the /_cluster/allocation/explain response
+area: Allocation
+type: enhancement
+issues: [97859]

docs/changelog/98622.yaml

@@ -0,0 +1,6 @@
+pr: 98622
+summary: Add 'dataset' size to cat indices and cat shards
+area: CAT APIs
+type: enhancement
+issues:
+ - 95092

docs/changelog/98628.yaml

@@ -0,0 +1,5 @@
+pr: 98628
+summary: Add ESQL own flavor of arithmetic operators
+area: ES|QL
+type: bug
+issues: []