Merge branch 'main' into downsample/cleanup_stuck_tasks

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -10,13 +10,13 @@
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
@@ -44,7 +44,7 @@ public static MapperService create(String mappings) {
         Settings settings = Settings.builder()
             .put("index.number_of_replicas", 0)
             .put("index.number_of_shards", 1)
-            .put("index.version.created", Version.CURRENT)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put("index.mapping.total_fields.limit", 10000)
             .build();
         IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.benchmark.routing.allocation;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -19,6 +18,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexVersion;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -128,7 +128,7 @@ public void setUp() throws Exception {
         for (int i = 1; i <= numIndices; i++) {
             mb.put(
                 IndexMetadata.builder("test_" + i)
-                    .settings(Settings.builder().put("index.version.created", Version.CURRENT))
+                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
                     .numberOfShards(numShards)
                     .numberOfReplicas(numReplicas)
             );

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.benchmark.routing.allocation;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -28,6 +27,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.health.HealthIndicatorResult;
 import org.elasticsearch.health.node.HealthInfo;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.TaskManager;
@@ -121,7 +121,7 @@ public void setUp() throws Exception {
             IndexMetadata indexMetadata = IndexMetadata.builder("test_" + i)
                 .settings(
                     Settings.builder()
-                        .put("index.version.created", Version.CURRENT)
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(DataTier.TIER_PREFERENCE_SETTING.getKey(), "data_warm")
                         .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm")
                 )

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -17,18 +17,17 @@
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
@@ -140,7 +139,6 @@ protected SearchExecutionContext buildSearchExecutionContext() {
             0,
             0,
             mapperService.getIndexSettings(),
-            ClusterSettings.createBuiltInClusterSettings(),
             null,
             (ft, fdc) -> ft.fielddataBuilder(fdc).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()),
             mapperService,
@@ -165,7 +163,7 @@ protected final MapperService createMapperService(String mappings) {
         Settings settings = Settings.builder()
             .put("index.number_of_replicas", 0)
             .put("index.number_of_shards", 1)
-            .put("index.version.created", Version.CURRENT)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .build();
         IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(meta, settings);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java

@@ -8,13 +8,17 @@
 
 package org.elasticsearch.gradle.internal;
 
+import org.elasticsearch.gradle.VersionProperties;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.TaskAction;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 
 import java.util.Collection;
 import java.util.stream.Collectors;
@@ -24,7 +28,12 @@
 import static org.elasticsearch.gradle.DistributionDownloadPlugin.DISTRO_EXTRACTED_CONFIG_PREFIX;
 import static org.elasticsearch.gradle.internal.test.rest.compat.compat.LegacyYamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME;
 
-public class ResolveAllDependencies extends DefaultTask {
+public abstract class ResolveAllDependencies extends DefaultTask {
+
+    private boolean resolveJavaToolChain = false;
+
+    @Inject
+    protected abstract JavaToolchainService getJavaToolchainService();
 
     private final ObjectFactory objectFactory;
 
@@ -43,7 +52,19 @@ public FileCollection getResolvedArtifacts() {
 
     @TaskAction
     void resolveAll() {
-        // do nothing, dependencies are resolved when snapshotting task inputs
+        if (resolveJavaToolChain) {
+            resolveDefaultJavaToolChain();
+        }
+    }
+
+    private void resolveDefaultJavaToolChain() {
+        getJavaToolchainService().launcherFor(javaToolchainSpec -> {
+            String bundledVendor = VersionProperties.getBundledJdkVendor();
+            String bundledJdkMajorVersion = VersionProperties.getBundledJdkMajorVersion();
+            javaToolchainSpec.getLanguageVersion().set(JavaLanguageVersion.of(bundledJdkMajorVersion));
+            javaToolchainSpec.getVendor()
+                .set(bundledVendor.equals("openjdk") ? JvmVendorSpec.ORACLE : JvmVendorSpec.matching(bundledVendor));
+        }).get();
     }
 
     @Internal
@@ -55,6 +76,15 @@ public void setConfigs(Collection<Configuration> configs) {
         this.configs = configs;
     }
 
+    @Internal
+    public boolean isResolveJavaToolChain() {
+        return resolveJavaToolChain;
+    }
+
+    public void setResolveJavaToolChain(boolean resolveJavaToolChain) {
+        this.resolveJavaToolChain = resolveJavaToolChain;
+    }
+
     private static boolean canBeResolved(Configuration configuration) {
         if (configuration.isCanBeResolved() == false) {
             return false;
@@ -67,4 +97,5 @@ private static boolean canBeResolved(Configuration configuration) {
         return configuration.getName().startsWith(DISTRO_EXTRACTED_CONFIG_PREFIX) == false
             && configuration.getName().equals(BWC_MINOR_CONFIG_NAME) == false;
     }
+
 }

build-tools-internal/version.properties

@@ -26,7 +26,7 @@ antlr4            = 4.11.1
 #  - x-pack/plugin/security
 bouncycastle=1.64
 # used by security and idp (need to be in sync due to cross-dependency in testing)
-opensaml = 4.0.1
+opensaml = 4.3.0
 
 # client dependencies
 httpclient        = 4.5.13

build.gradle

@@ -200,6 +200,7 @@ allprojects {
 
   tasks.register('resolveAllDependencies', ResolveAllDependencies) {
     configs = project.configurations
+    resolveJavaToolChain = true
     if (project.path.contains("fixture")) {
       dependsOn tasks.withType(ComposePull)
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java

@@ -219,7 +219,8 @@ private static Process createProcess(
         command.add(esHome.resolve("lib").toString());
         // Special circumstances require some modules (not depended on by the main server module) to be explicitly added:
         command.add("--add-modules=jdk.net"); // needed to reflectively set extended socket options
-        command.add("--add-modules=org.elasticsearch.preallocate"); // needed on boot layer as target to open java.io
+        // we control the module path, which may have additional modules not required by server
+        command.add("--add-modules=ALL-MODULE-PATH");
         command.add("-m");
         command.add("org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch");
 

docs/changelog/97409.yaml

@@ -0,0 +1,5 @@
+pr: 97409
+summary: Trim stored fields for `_id` field in tsdb
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/97972.yaml

@@ -0,0 +1,6 @@
+pr: 97972
+summary: Automatically flatten objects when subobjects:false
+area: Mapping
+type: enhancement
+issues:
+ - 88934

docs/changelog/98550.yaml

@@ -0,0 +1,5 @@
+pr: 98550
+summary: Report a node's "roles" setting in the /_cluster/allocation/explain response
+area: Allocation
+type: enhancement
+issues: [97859]

docs/changelog/98654.yaml

@@ -0,0 +1,5 @@
+pr: 98654
+summary: Allow native users/roles to be disabled via setting
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/98692.yaml

@@ -0,0 +1,5 @@
+pr: 98692
+summary: Add request header size limit for RCS transport connections
+area: Network
+type: enhancement
+issues: []

docs/changelog/98759.yaml

@@ -0,0 +1,6 @@
+pr: 98759
+summary: "ESQL: Support queries that don't return underlying fields"
+area: ES|QL
+type: bug
+issues:
+ - 98404

docs/changelog/98765.yaml

@@ -0,0 +1,6 @@
+pr: 98765
+summary: Fix mvt error when returning partial results
+area: Geo
+type: bug
+issues:
+ - 98730

docs/changelog/98808.yaml

@@ -0,0 +1,6 @@
+pr: 98808
+summary: Set default index mode for `TimeSeries` to `null`
+area: Aggregations
+type: enhancement
+issues:
+ - 97429

docs/changelog/98809.yaml

@@ -0,0 +1,7 @@
+pr: 98809
+summary: Avoiding the use of nodes that are no longer in the cluster when computing
+  master stability
+area: Health
+type: enhancement
+issues:
+ - 98636

docs/changelog/98811.yaml

@@ -0,0 +1,5 @@
+pr: 98811
+summary: Allow explain data stream lifecycle to accept a data stream
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/98844.yaml

@@ -0,0 +1,5 @@
+pr: 98844
+summary: Add accessors required to recreate `TransformStats` object from the fields
+area: Transform
+type: enhancement
+issues: []

docs/reference/ccr/getting-started.asciidoc

@@ -159,7 +159,7 @@ cluster with cluster alias `leader`.
     connected to.
 ====
 
-include::../../../x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc[tag=configure-ccr-privileges]
+include::../../../x-pack/docs/en/security/authentication/remote-clusters-privileges-cert.asciidoc[tag=configure-ccr-privileges]
 
 [[ccr-getting-started-follower-index]]
 ==== Create a follower index to replicate a specific index