Merge branch 'main' into carlosdelest/semantic-text-new-ingestion-inf…

…erence

TESTING.asciidoc

@@ -551,13 +551,19 @@ When running `./gradlew check`, minimal bwc checks are also run against compatib
 
 ==== BWC Testing against a specific remote/branch
 
-Sometimes a backward compatibility change spans two versions. A common case is a new functionality
-that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
-To test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of
-pulling the release branch from GitHub. You do so using the `bwc.remote` and `bwc.refspec.BRANCH` system properties:
+Sometimes a backward compatibility change spans two versions.
+A common case is a new functionality that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
+Another use case, since the introduction of serverless, is to test BWC against main in addition to the other released branches.
+To do so, specify the `bwc.refspec` remote and branch to use for the BWC build as `origin/main`.
+To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java], 
+increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp 
+in link:./x-pack/plugin/ml/build.gradle[ml/build.gradle].
+
+In general, to test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of pulling the release branch from GitHub.
+You do so using the `bwc.refspec.{VERSION}` system property:
 
 -------------------------------------------------
-./gradlew check -Dbwc.remote=${remote} -Dbwc.refspec.5.x=index_req_bwc_5.x
+./gradlew check -Dtests.bwc.refspec.8.15=origin/main
 -------------------------------------------------
 
 The branch needs to be available on the remote that the BWC makes of the

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
@@ -71,7 +72,8 @@ public static MapperService create(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            MapperMetrics.NOOP
         );
 
         try {

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -154,7 +155,8 @@ protected SearchExecutionContext buildSearchExecutionContext() {
             null,
             () -> true,
             null,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            MapperMetrics.NOOP
         );
     }
 
@@ -186,7 +188,8 @@ protected final MapperService createMapperService(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            MapperMetrics.NOOP
         );
 
         try {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java

@@ -60,6 +60,7 @@
 import static org.gradle.api.JavaVersion.VERSION_20;
 import static org.gradle.api.JavaVersion.VERSION_21;
 import static org.gradle.api.JavaVersion.VERSION_22;
+import static org.gradle.api.JavaVersion.VERSION_23;
 
 @CacheableTask
 public abstract class ThirdPartyAuditTask extends DefaultTask {
@@ -336,8 +337,8 @@ private String runForbiddenAPIsCli() throws IOException {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
             spec.classpath(getForbiddenAPIsClasspath(), classpath);
-            // Enable explicitly for each release as appropriate. Just JDK 20/21/22 for now, and just the vector module.
-            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22)) {
+            // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
+            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
             }
             spec.jvmArgs("-Xmx1g");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java

@@ -11,7 +11,6 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import org.apache.commons.compress.utils.Lists;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainDownload;
 import org.gradle.jvm.toolchain.JavaToolchainRequest;
@@ -21,25 +20,25 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
-import java.util.Comparator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.StreamSupport;
 
 import static org.gradle.jvm.toolchain.JavaToolchainDownload.fromUri;
 
 public abstract class AdoptiumJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
     // package protected for better testing
-    final Map<AdoptiumVersionRequest, Optional<AdoptiumVersionInfo>> CACHED_SEMVERS = new ConcurrentHashMap<>();
+    final Map<AdoptiumVersionRequest, Optional<String>> CACHED_RELEASES = new ConcurrentHashMap<>();
 
     @Override
     public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
         if (requestIsSupported(request) == false) {
             return Optional.empty();
         }
         AdoptiumVersionRequest versionRequestKey = toVersionRequest(request);
-        Optional<AdoptiumVersionInfo> versionInfo = CACHED_SEMVERS.computeIfAbsent(
+        Optional<String> versionInfo = CACHED_RELEASES.computeIfAbsent(
             versionRequestKey,
             (r) -> resolveAvailableVersion(versionRequestKey)
         );
@@ -54,12 +53,12 @@ private AdoptiumVersionRequest toVersionRequest(JavaToolchainRequest request) {
         return new AdoptiumVersionRequest(platform, arch, javaLanguageVersion);
     }
 
-    private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
+    private Optional<String> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
         ObjectMapper mapper = new ObjectMapper();
         try {
             int languageVersion = requestKey.languageVersion.asInt();
             URL source = new URL(
-                "https://api.adoptium.net/v3/info/release_versions?architecture="
+                "https://api.adoptium.net/v3/info/release_names?architecture="
                     + requestKey.arch
                     + "&image_type=jdk&os="
                     + requestKey.platform
@@ -71,14 +70,8 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
                     + ")"
             );
             JsonNode jsonNode = mapper.readTree(source);
-            JsonNode versionsNode = jsonNode.get("versions");
-            return Optional.of(
-                Lists.newArrayList(versionsNode.iterator())
-                    .stream()
-                    .map(this::toVersionInfo)
-                    .max(Comparator.comparing(AdoptiumVersionInfo::semver))
-                    .get()
-            );
+            JsonNode versionsNode = jsonNode.get("releases");
+            return StreamSupport.stream(versionsNode.spliterator(), false).map(JsonNode::textValue).findFirst();
         } catch (FileNotFoundException e) {
             // request combo not supported (e.g. aarch64 + windows
             return Optional.empty();
@@ -87,21 +80,10 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
         }
     }
 
-    private AdoptiumVersionInfo toVersionInfo(JsonNode node) {
-        return new AdoptiumVersionInfo(
-            node.get("build").asInt(),
-            node.get("major").asInt(),
-            node.get("minor").asInt(),
-            node.get("openjdk_version").asText(),
-            node.get("security").asInt(),
-            node.get("semver").asText()
-        );
-    }
-
-    private URI resolveDownloadURI(AdoptiumVersionRequest request, AdoptiumVersionInfo versionInfo) {
+    private URI resolveDownloadURI(AdoptiumVersionRequest request, String version) {
         return URI.create(
-            "https://api.adoptium.net/v3/binary/version/jdk-"
-                + versionInfo.semver
+            "https://api.adoptium.net/v3/binary/version/"
+                + version
                 + "/"
                 + request.platform
                 + "/"
@@ -118,7 +100,5 @@ private boolean requestIsSupported(JavaToolchainRequest request) {
         return anyVendorOr(request.getJavaToolchainSpec().getVendor().get(), JvmVendorSpec.ADOPTIUM);
     }
 
-    record AdoptiumVersionInfo(int build, int major, int minor, String openjdkVersion, int security, String semver) {}
-
     record AdoptiumVersionRequest(String platform, String arch, JavaLanguageVersion languageVersion) {}
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java

@@ -39,11 +39,7 @@ record JdkBuild(JavaLanguageVersion languageVersion, String version, String buil
     );
 
     // package private so it can be replaced by tests
-    List<JdkBuild> builds = List.of(
-        getBundledJdkBuild(),
-        // 22 release candidate
-        new JdkBuild(JavaLanguageVersion.of(22), "22", "36", "830ec9fcccef480bb3e73fb7ecafe059")
-    );
+    List<JdkBuild> builds = List.of(getBundledJdkBuild());
 
     private JdkBuild getBundledJdkBuild() {
         String bundledJdkVersion = VersionProperties.getBundledJdkVersion();

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy

@@ -11,7 +11,6 @@ package org.elasticsearch.gradle.internal.toolchain
 import org.gradle.api.services.BuildServiceParameters
 import org.gradle.jvm.toolchain.JavaLanguageVersion
 import org.gradle.jvm.toolchain.JavaToolchainResolver
-import org.gradle.platform.OperatingSystem
 
 import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toArchString
 import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toOsString
@@ -38,12 +37,7 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
                     toOsString(it[2], it[1]),
                     toArchString(it[3]),
                     languageVersion);
-            resolver.CACHED_SEMVERS.put(request, Optional.of(new AdoptiumJdkToolchainResolver.AdoptiumVersionInfo(languageVersion.asInt(),
-                    1,
-                    1,
-                    "" + languageVersion.asInt() + ".1.1.1+37",
-                    0, "" + languageVersion.asInt() + ".1.1.1+37.1"
-            )))
+            resolver.CACHED_RELEASES.put(request, Optional.of('jdk-' + languageVersion.asInt() + '.1.1.1+37.1'))
 
         }
         return resolver

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 8.15.0
 lucene            = 9.10.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac
+bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1107,11 +1107,11 @@ private void logFileContents(String description, Path from, boolean tailLogs) {
             return;
         }
 
-        boolean foundNettyLeaks = false;
+        boolean foundLeaks = false;
         for (String logLine : errorsAndWarnings.keySet()) {
-            if (logLine.contains("ResourceLeakDetector]")) {
+            if (logLine.contains("ResourceLeakDetector") || logLine.contains("LeakTracker")) {
                 tailLogs = true;
-                foundNettyLeaks = true;
+                foundLeaks = true;
                 break;
             }
         }
@@ -1140,8 +1140,8 @@ private void logFileContents(String description, Path from, boolean tailLogs) {
                 });
             }
         }
-        if (foundNettyLeaks) {
-            throw new TestClustersException("Found Netty ByteBuf leaks in node logs.");
+        if (foundLeaks) {
+            throw new TestClustersException("Found resource leaks in node logs.");
         }
     }
 

client/test/build.gradle

@@ -27,9 +27,9 @@ dependencies {
   api "org.hamcrest:hamcrest:${versions.hamcrest}"
 
   // mockito
-  api 'org.mockito:mockito-core:5.9.0'
-  api 'org.mockito:mockito-subclass:5.9.0'
-  api 'net.bytebuddy:byte-buddy:1.14.11'
+  api 'org.mockito:mockito-core:5.11.0'
+  api 'org.mockito:mockito-subclass:5.11.0'
+  api 'net.bytebuddy:byte-buddy:1.14.12'
   api 'org.objenesis:objenesis:3.3'
 }