Merge remote-tracking branch 'origin/main' into esql_searchstats_for_…

…indexed_docvalues

.buildkite/packer_cache.sh

@@ -30,5 +30,7 @@ for branch in "${branches[@]}"; do
 
   export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
   "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI --max-workers=4
+  "checkout/${branch}/gradlew" --stop
+  pkill -f '.*GradleDaemon.*'
   rm -rf "checkout/${branch}"
 done

.buildkite/scripts/cloud-deploy.sh

@@ -2,11 +2,11 @@
 
 set -euo pipefail
 
-.ci/scripts/run-gradle.sh buildCloudDockerImage
+.ci/scripts/run-gradle.sh buildCloudEssDockerImage
 
 ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}')
-DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}"
-docker tag elasticsearch-cloud:test "$DOCKER_TAG"
+DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud-ess:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}"
+docker tag elasticsearch-cloud-ess:test "$DOCKER_TAG"
 
 echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin docker.elastic.co
 unset DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD

benchmarks/build.gradle

@@ -40,15 +40,15 @@ dependencies {
     // us to invoke the JMH uberjar as usual.
     exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
   }
-  api(project(':libs:elasticsearch-h3'))
+  api(project(':libs:h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
-  implementation project(path: ':libs:elasticsearch-simdvec')
+  implementation project(path: ':libs:simdvec')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
-  nativeLib(project(':libs:elasticsearch-native'))
+  nativeLib(project(':libs:native'))
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy

@@ -18,7 +18,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
         // required for JarHell to work
-        subProject(":libs:elasticsearch-core") << "apply plugin:'java'"
+        subProject(":libs:core") << "apply plugin:'java'"
 
         configurationCacheCompatible = false
     }

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -161,7 +161,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       ':server:generateModulesList',
       ':server:generatePluginsList',
       ':generateProviderImpls',
-      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs',
+      ':libs:native:native-libraries:extractLibs',
       ':x-pack:libs:es-opensaml-security-api:shadowJar'].collect { elasticsearchProject.right()?.task(it) ?: it })
   }
 

build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle

@@ -33,12 +33,12 @@ BuildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ex
     if (unreleasedVersion) {
       // For unreleased snapshot versions, build them from source
       "oldJar${baseName}"(files(project(unreleasedVersion.gradleProjectPath).tasks.named(buildBwcTaskName(project.name))))
-    } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("elasticsearch-logging")==false) {
+    } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("logging")==false) {
       //there was a package rename in 8.7.0, except for es-logging
-      "oldJar${baseName}"("org.elasticsearch.plugin:${project.name}:${bwcVersion}")
+      "oldJar${baseName}"("org.elasticsearch.plugin:elasticsearch-${project.name}:${bwcVersion}")
     } else {
       // For released versions, download it
-      "oldJar${baseName}"("org.elasticsearch:${project.name}:${bwcVersion}")
+      "oldJar${baseName}"("org.elasticsearch:elasticsearch-${project.name}:${bwcVersion}")
     }
   }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -24,7 +24,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bf163e1977002301f7b9fd28fe6837a8cb2dd5c83e4cd45fb67fb28d15d5d40f",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:973431347ad45f40e01afbbd010bf9de929c088a63382239b90dd84f39618bc8",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -177,7 +177,7 @@ public static void configureInputNormalization(Project project) {
     }
 
     private static void configureNativeLibraryPath(Project project) {
-        String nativeProject = ":libs:elasticsearch-native:elasticsearch-native-libraries";
+        String nativeProject = ":libs:native:native-libraries";
         Configuration nativeConfig = project.getConfigurations().create("nativeLibs");
         nativeConfig.defaultDependencies(deps -> {
             deps.add(project.getDependencies().project(Map.of("path", nativeProject, "configuration", "default")));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -165,7 +165,12 @@ private static void configureBwcProject(
             DistributionProjectArtifact stableAnalysisPluginProjectArtifact = new DistributionProjectArtifact(
                 new File(
                     checkoutDir.get(),
-                    relativeDir + "/build/distributions/" + stableApiProject.getName() + "-" + bwcVersion.get() + "-SNAPSHOT.jar"
+                    relativeDir
+                        + "/build/distributions/elasticsearch-"
+                        + stableApiProject.getName()
+                        + "-"
+                        + bwcVersion.get()
+                        + "-SNAPSHOT.jar"
                 ),
                 null
             );
@@ -275,7 +280,7 @@ private static List<DistributionProject> resolveArchiveProjects(File checkoutDir
     }
 
     private static List<Project> resolveStableProjects(Project project) {
-        Set<String> stableProjectNames = Set.of("elasticsearch-logging", "elasticsearch-plugin-api", "elasticsearch-plugin-analysis-api");
+        Set<String> stableProjectNames = Set.of("logging", "plugin-api", "plugin-analysis-api");
         return project.findProject(":libs")
             .getSubprojects()
             .stream()
@@ -312,7 +317,9 @@ static void createBuildBwcTask(
                 c.getOutputs().files(expectedOutputFile);
             }
             c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false);
-            c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
+            c.getArgs().add("-p");
+            c.getArgs().add(projectPath);
+            c.getArgs().add(assembleTaskName);
             if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
                 c.getArgs().add("--build-cache");
             }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java

@@ -21,11 +21,11 @@ public class JarHellPrecommitPlugin extends PrecommitPlugin {
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPluginManager().apply(JarHellPlugin.class);
 
-        if (project.getPath().equals(":libs:elasticsearch-core") == false) {
+        if (project.getPath().equals(":libs:core") == false) {
             // ideally we would configure this as a default dependency. But Default dependencies do not work correctly
             // with gradle project dependencies as they're resolved to late in the build and don't setup according task
             // dependencies properly
-            var elasticsearchCoreProject = project.findProject(":libs:elasticsearch-core");
+            var elasticsearchCoreProject = project.findProject(":libs:core");
             if (elasticsearchCoreProject != null) {
                 project.getDependencies().add("jarHell", elasticsearchCoreProject);
             }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java

@@ -27,7 +27,7 @@
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
 
     public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell";
-    public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:elasticsearch-core";
+    public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:core";
 
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {

build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -56,7 +56,7 @@ abstract class AbstractGradleFuncTest extends Specification {
         propertiesFile <<
             "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME"
 
-        def nativeLibsProject = subProject(":libs:elasticsearch-native:elasticsearch-native-libraries")
+        def nativeLibsProject = subProject(":libs:native:native-libraries")
         nativeLibsProject << """
             plugins {
                 id 'base'

client/rest/build.gradle

@@ -79,7 +79,7 @@ tasks.named('forbiddenApisTest').configure {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:elasticsearch-core
+// TODO: Not anymore. Now in :libs:core
 tasks.named("jarHell").configure {
   enabled = false
 }

client/sniffer/build.gradle

@@ -73,7 +73,7 @@ tasks.named("dependencyLicenses").configure {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:elasticsearch-core
+// TODO: Not anymore. Now in :libs:core
 tasks.named("jarHell").configure { enabled = false }
 
 tasks.named("testTestingConventions").configure {

client/test/build.gradle

@@ -54,7 +54,7 @@ tasks.named('forbiddenApisTest').configure {
 tasks.named("thirdPartyAudit").configure { enabled = false }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:elasticsearch-core
+// TODO: Not anymore. Now in :libs:core
 tasks.named("jarHell").configure { enabled = false }
 
 // TODO: should we have licenses for our test deps?

distribution/build.gradle

@@ -275,7 +275,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     }
     all {
       resolutionStrategy.dependencySubstitution {
-        substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:elasticsearch-log4j") because "patched to remove JndiLookup clas"}
+        substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:log4j") because "patched to remove JndiLookup clas"}
     }
   }
 
@@ -291,7 +291,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
-    libsNative project(':libs:elasticsearch-native:elasticsearch-native-libraries')
+    libsNative project(':libs:native:native-libraries')
   }
 
   project.ext {

distribution/tools/entitlement-agent/build.gradle

@@ -22,7 +22,7 @@ configurations {
 
 dependencies {
   entitlementBridge project(":distribution:tools:entitlement-bridge")
-  compileOnly project(":libs:elasticsearch-core")
+  compileOnly project(":libs:core")
   compileOnly project(":distribution:tools:entitlement-runtime")
   testImplementation project(":test:framework")
   testImplementation project(":distribution:tools:entitlement-bridge")

distribution/tools/entitlement-runtime/build.gradle

@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 
 dependencies {
-  compileOnly project(':libs:elasticsearch-core') // For @SuppressForbidden
-  compileOnly project(":libs:elasticsearch-x-content") // for parsing policy files
+  compileOnly project(':libs:core') // For @SuppressForbidden
+  compileOnly project(":libs:x-content") // for parsing policy files
   compileOnly project(':server') // To access the main server module for special permission checks
   compileOnly project(':distribution:tools:entitlement-bridge')
   testImplementation project(":test:framework")

distribution/tools/geoip-cli/build.gradle

@@ -15,8 +15,8 @@ base {
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
-  compileOnly project(":libs:elasticsearch-x-content")
+  compileOnly project(":libs:cli")
+  compileOnly project(":libs:x-content")
   testImplementation project(":test:framework")
   testImplementation "org.apache.commons:commons-compress:1.26.1"
   testImplementation "commons-io:commons-io:2.15.1"

distribution/tools/keystore-cli/build.gradle

@@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
+  compileOnly project(":libs:cli")
   testImplementation project(":test:framework")
   testImplementation "com.google.jimfs:jimfs:${versions.jimfs}"
   testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}"

distribution/tools/plugin-cli/build.gradle

@@ -21,9 +21,9 @@ tasks.named("dependencyLicenses").configure {
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
-  implementation project(":libs:elasticsearch-plugin-api")
-  implementation project(":libs:elasticsearch-plugin-scanner")
+  compileOnly project(":libs:cli")
+  implementation project(":libs:plugin-api")
+  implementation project(":libs:plugin-scanner")
   // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice
   implementation 'org.ow2.asm:asm:9.7'
   implementation 'org.ow2.asm:asm-tree:9.7'

distribution/tools/server-cli/build.gradle

@@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
+  compileOnly project(":libs:cli")
 
   testImplementation project(":test:framework")
 }

distribution/tools/windows-service-cli/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
+  compileOnly project(":libs:cli")
   compileOnly project(":distribution:tools:server-cli")
 
   testImplementation project(":test:framework")

docs/changelog/115266.yaml

@@ -0,0 +1,6 @@
+pr: 115266
+summary: ES|QL CCS uses `skip_unavailable` setting for handling disconnected remote
+  clusters
+area: ES|QL
+type: enhancement
+issues: [ 114531 ]

docs/changelog/115807.yaml

@@ -0,0 +1,5 @@
+pr: 115807
+summary: "[Inference API] Improve chunked results error message"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/115823.yaml

@@ -0,0 +1,5 @@
+pr: 115823
+summary: Add ECK Role Mapping Cleanup
+area: Security
+type: bug
+issues: []

docs/changelog/115831.yaml

@@ -0,0 +1,13 @@
+pr: 115831
+summary: Increase minimum threshold in shard balancer
+area: Allocation
+type: breaking
+issues: []
+breaking:
+  title: Minimum shard balancer threshold is now 1.0
+  area: Cluster and node setting
+  details: >-
+    Earlier versions of {es} accepted any non-negative value for `cluster.routing.allocation.balance.threshold`, but values smaller than
+    `1.0` do not make sense and have been ignored since version 8.6.1. From 9.0.0 these nonsensical values are now forbidden.
+  impact: Do not set `cluster.routing.allocation.balance.threshold` to a value less than `1.0`.
+  notable: false

docs/changelog/115834.yaml

@@ -0,0 +1,5 @@
+pr: 115834
+summary: Try to simplify geometries that fail with `TopologyException`
+area: Geo
+type: bug
+issues: []

docs/changelog/115836.yaml

@@ -0,0 +1,5 @@
+pr: 115836
+summary: Catch and handle disconnect exceptions in search
+area: Search
+type: bug
+issues: []

docs/reference/landing-page.asciidoc

@@ -128,7 +128,7 @@
       <a href="https://www.elastic.co/guide/en/cloud/current/ec-cloud-ingest-data.html">Adding data to Elasticsearch</a>
     </li>
     <li>
-      <a href="https://www.elastic.co/guide/en/enterprise-search/current/connectors.html">Connectors</a>
+      <a href="es-connectors.html">Connectors</a>
     </li>
     <li>
       <a href="https://www.elastic.co/guide/en/enterprise-search/current/crawler.html">Web crawler</a>

libs/build.gradle

@@ -7,10 +7,42 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-configure(childProjects.values() - project('elasticsearch-log4j')) {
+configure(childProjects.values()) {
+
+  apply plugin: 'base'
+
   /*
-   * All subprojects are java projects using Elasticsearch's standard build
-   * tools.
+   * Although these libs are local to Elasticsearch, they can conflict with other similarly
+   * named libraries when downloaded into a single directory via maven. Here we set the
+   * name of all libs to begin with the "elasticsearch-" prefix. Additionally, subprojects
+   * of libs begin with their parents artifactId.
    */
-  apply plugin: 'elasticsearch.build'
+  def baseProject = project
+  def baseArtifactId = "elasticsearch-${it.name}"
+  base {
+    archivesName = baseArtifactId
+  }
+  subprojects {
+    apply plugin: 'base'
+
+    def subArtifactId = baseArtifactId
+    def currentProject = project
+    while (currentProject != baseProject) {
+      subArtifactId += "-${currentProject.name}"
+      currentProject = currentProject.parent
+    }
+    base {
+      archivesName = subArtifactId
+    }
+  }
+
+  // log4j is a hack, and not really a full elasticsearch built jar
+  if (project.name != 'log4j') {
+
+    /*
+     * All subprojects are java projects using Elasticsearch's standard build
+     * tools.
+     */
+    apply plugin: 'elasticsearch.build'
+  }
 }

libs/cli/build.gradle

@@ -11,10 +11,10 @@ apply plugin: 'elasticsearch.publish'
 
 dependencies {
   api 'net.sf.jopt-simple:jopt-simple:5.0.2'
-  api project(':libs:elasticsearch-core')
+  api project(':libs:core')
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-cli'
+    exclude group: 'org.elasticsearch', module: 'cli'
   }
 }