Merge branch 'main' into esql-add-sort-to-some-tests

.buildkite/pipelines/periodic-packaging.template.yml

@@ -10,9 +10,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/periodic-packaging.yml

@@ -11,9 +11,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/periodic-platform-support.yml

@@ -10,9 +10,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -13,9 +13,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/pull-request/part-1.yml

@@ -1,6 +1,8 @@
 steps:
   - label: part-1
-    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1
+    command: |
+      .buildkite/scripts/spotless.sh # This doesn't have to be part of part-1, it was just a convenient place to put it
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1
     timeout_in_minutes: 300
     agents:
       provider: gcp

.buildkite/pipelines/pull-request/precommit.yml

@@ -3,7 +3,9 @@ config:
   skip-labels: []
 steps:
   - label: precommit
-    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
+    command: |
+      .buildkite/scripts/spotless.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
     timeout_in_minutes: 300
     agents:
       provider: gcp

.buildkite/scripts/gradle-configuration-cache-validation.sh

@@ -3,16 +3,16 @@
 set -euo pipefail
 
 # This is a workaround for https://github.com/gradle/gradle/issues/28159
-.ci/scripts/run-gradle.sh --no-daemon precommit
+.ci/scripts/run-gradle.sh --no-daemon $@
 
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@
 
 # Create a temporary file
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
 echo "2nd run"
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ | tee $tmpOutputFile
 
 # Check if the command was successful
 if grep -q "Configuration cache entry reused." $tmpOutputFile; then

.buildkite/scripts/spotless.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+  echo "Not a pull request, skipping spotless"
+  exit 0
+fi
+
+if ! git diff --exit-code; then
+  echo "Changes are present before running spotless, not running"
+  git status
+  exit 0
+fi
+
+NEW_COMMIT_MESSAGE="[CI] Auto commit changes from spotless"
+PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+echo "--- Running spotless"
+.ci/scripts/run-gradle.sh -Dscan.tag.NESTED spotlessApply
+
+if git diff --exit-code; then
+  echo "No changes found after running spotless. Don't need to auto commit."
+  exit 0
+fi
+
+if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+  echo "Changes found after running spotless"
+  echo "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+  echo "Please review and fix manually."
+  exit 1
+fi
+
+git config --global user.name elasticsearchmachine
+git config --global user.email '[email protected]'
+
+gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+git add -u .
+git commit -m "$NEW_COMMIT_MESSAGE"
+git push
+
+# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+# So, let's just sleep to give the build time to cancel itself without an error
+# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+sleep 300
+exit 1

.ci/scripts/run-gradle.sh

@@ -5,35 +5,39 @@ mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
 
 MAX_WORKERS=4
 
-# Don't run this stuff on Windows
-if ! uname -a | grep -q MING; then
-  # drop page cache and kernel slab objects on linux
-  [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+if [ -z "$MAX_WORKER_ENV" ]; then
+    # Don't run this stuff on Windows
+    if ! uname -a | grep -q MING; then
+      # drop page cache and kernel slab objects on linux
+      [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
-  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-    MAX_WORKERS=16
-  elif [ -f /proc/cpuinfo ]; then
-    MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
-  else
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
-    else
-        echo "Unsupported OS Type: $OSTYPE"
-        exit 1
-    fi
-  fi
-  if pwd | grep -v -q ^/dev/shm ; then
-    echo "Not running on a ramdisk, reducing number of workers"
-    MAX_WORKERS=$(($MAX_WORKERS*2/3))
-  fi
+      if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+        MAX_WORKERS=16
+      elif [ -f /proc/cpuinfo ]; then
+        MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+      else
+        if [[ "$OSTYPE" == "darwin"* ]]; then
+            MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+        else
+            echo "Unsupported OS Type: $OSTYPE"
+            exit 1
+        fi
+      fi
+      if pwd | grep -v -q ^/dev/shm ; then
+        echo "Not running on a ramdisk, reducing number of workers"
+        MAX_WORKERS=$(($MAX_WORKERS*2/3))
+      fi
 
-  # Export glibc version as environment variable since some BWC tests are incompatible with later versions
-  export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
-fi
+      # Export glibc version as environment variable since some BWC tests are incompatible with later versions
+      export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+    fi
 
-# Running on 2-core machines without ramdisk can make this value be 0
-if [[ "$MAX_WORKERS" == "0" ]]; then
-  MAX_WORKERS=1
+    # Running on 2-core machines without ramdisk can make this value be 0
+    if [[ "$MAX_WORKERS" == "0" ]]; then
+      MAX_WORKERS=1
+    fi
+else
+    MAX_WORKERS=$MAX_WORKER_ENV
 fi
 
 set -e

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -16,6 +16,7 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
+import org.gradle.api.Task;
 import org.gradle.api.logging.LogLevel;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
@@ -103,10 +104,16 @@ private static TaskProvider<LoggedExec> createRunBwcGradleTask(
             loggedExec.dependsOn("checkoutBwcBranch");
             loggedExec.getWorkingDir().set(checkoutDir.get());
 
-            loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> {
-                spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
-                spec.getParameters().getCheckoutDir().set(checkoutDir);
-            }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))));
+            loggedExec.doFirst(new Action<Task>() {
+                @Override
+                public void execute(Task task) {
+                    Provider<String> minimumCompilerVersionValueSource = providerFactory.of(JavaHomeValueSource.class, spec -> {
+                        spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
+                        spec.getParameters().getCheckoutDir().set(checkoutDir);
+                    }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)));
+                    loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", minimumCompilerVersionValueSource.get());
+                }
+            });
 
             if (isCi && OS.current() != OS.WINDOWS) {
                 // TODO: Disabled for now until we can figure out why files are getting corrupted
@@ -169,14 +176,6 @@ private static Provider<String> getJavaHome(ObjectFactory objectFactory, JavaToo
             .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
     }
 
-    private static String readFromFile(File file) {
-        try {
-            return FileUtils.readFileToString(file).trim();
-        } catch (IOException ioException) {
-            throw new GradleException("Cannot read java properties file.", ioException);
-        }
-    }
-
     public abstract static class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {
 
         private String minimumCompilerVersionPath(Version bwcVersion) {
@@ -185,6 +184,14 @@ private String minimumCompilerVersionPath(Version bwcVersion) {
                 : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
         }
 
+        private static String readFromFile(File file) {
+            try {
+                return FileUtils.readFileToString(file).trim();
+            } catch (IOException ioException) {
+                throw new GradleException("Cannot read java properties file.", ioException);
+            }
+        }
+
         @Override
         public String obtain() {
             return readFromFile(

docs/changelog/118454.yaml

@@ -0,0 +1,5 @@
+pr: 118454
+summary: Fix RLIKE folding with (unsupported) case insensitive pattern
+area: ES|QL
+type: bug
+issues: []

docs/changelog/118704.yaml

@@ -0,0 +1,6 @@
+pr: 118704
+summary: Avoid updating settings version in `MetadataMigrateToDataStreamService` when
+  settings have not changed
+area: Data streams
+type: bug
+issues: []

libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java

@@ -29,15 +29,13 @@
 public class InstrumentationServiceImpl implements InstrumentationService {
 
     @Override
-    public Instrumenter newInstrumenter(Map<MethodKey, CheckMethod> checkMethods) {
-        return InstrumenterImpl.create(checkMethods);
+    public Instrumenter newInstrumenter(Class<?> clazz, Map<MethodKey, CheckMethod> methods) {
+        return InstrumenterImpl.create(clazz, methods);
     }
 
     @Override
-    public Map<MethodKey, CheckMethod> lookupMethodsToInstrument(String entitlementCheckerClassName) throws ClassNotFoundException,
-        IOException {
+    public Map<MethodKey, CheckMethod> lookupMethods(Class<?> checkerClass) throws IOException {
         var methodsToInstrument = new HashMap<MethodKey, CheckMethod>();
-        var checkerClass = Class.forName(entitlementCheckerClassName);
         var classFileInfo = InstrumenterImpl.getClassFileInfo(checkerClass);
         ClassReader reader = new ClassReader(classFileInfo.bytecodes());
         ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) {

libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java

@@ -58,30 +58,14 @@ public class InstrumenterImpl implements Instrumenter {
         this.checkMethods = checkMethods;
     }
 
-    static String getCheckerClassName() {
-        int javaVersion = Runtime.version().feature();
-        final String classNamePrefix;
-        if (javaVersion >= 23) {
-            classNamePrefix = "Java23";
-        } else {
-            classNamePrefix = "";
-        }
-        return "org/elasticsearch/entitlement/bridge/" + classNamePrefix + "EntitlementChecker";
-    }
-
-    public static InstrumenterImpl create(Map<MethodKey, CheckMethod> checkMethods) {
-        String checkerClass = getCheckerClassName();
-        String handleClass = checkerClass + "Handle";
-        String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(Type.getObjectType(checkerClass));
+    public static InstrumenterImpl create(Class<?> checkerClass, Map<MethodKey, CheckMethod> checkMethods) {
+        Type checkerClassType = Type.getType(checkerClass);
+        String handleClass = checkerClassType.getInternalName() + "Handle";
+        String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(checkerClassType);
         return new InstrumenterImpl(handleClass, getCheckerClassMethodDescriptor, "", checkMethods);
     }
 
-    public ClassFileInfo instrumentClassFile(Class<?> clazz) throws IOException {
-        ClassFileInfo initial = getClassFileInfo(clazz);
-        return new ClassFileInfo(initial.fileName(), instrumentClass(Type.getInternalName(clazz), initial.bytecodes()));
-    }
-
-    public static ClassFileInfo getClassFileInfo(Class<?> clazz) throws IOException {
+    static ClassFileInfo getClassFileInfo(Class<?> clazz) throws IOException {
         String internalName = Type.getInternalName(clazz);
         String fileName = "/" + internalName + ".class";
         byte[] originalBytecodes;
@@ -306,5 +290,5 @@ protected void pushEntitlementChecker(MethodVisitor mv) {
         mv.visitMethodInsn(INVOKESTATIC, handleClass, "instance", getCheckerClassMethodDescriptor, false);
     }
 
-    public record ClassFileInfo(String fileName, byte[] bytecodes) {}
+    record ClassFileInfo(String fileName, byte[] bytecodes) {}
 }

libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java

@@ -51,8 +51,8 @@ interface TestCheckerCtors {
         void check$org_example_TestTargetClass$(Class<?> clazz, int x, String y);
     }
 
-    public void testInstrumentationTargetLookup() throws IOException, ClassNotFoundException {
-        Map<MethodKey, CheckMethod> checkMethods = instrumentationService.lookupMethodsToInstrument(TestChecker.class.getName());
+    public void testInstrumentationTargetLookup() throws IOException {
+        Map<MethodKey, CheckMethod> checkMethods = instrumentationService.lookupMethods(TestChecker.class);
 
         assertThat(checkMethods, aMapWithSize(3));
         assertThat(
@@ -116,8 +116,8 @@ public void testInstrumentationTargetLookup() throws IOException, ClassNotFoundE
         );
     }
 
-    public void testInstrumentationTargetLookupWithOverloads() throws IOException, ClassNotFoundException {
-        Map<MethodKey, CheckMethod> checkMethods = instrumentationService.lookupMethodsToInstrument(TestCheckerOverloads.class.getName());
+    public void testInstrumentationTargetLookupWithOverloads() throws IOException {
+        Map<MethodKey, CheckMethod> checkMethods = instrumentationService.lookupMethods(TestCheckerOverloads.class);
 
         assertThat(checkMethods, aMapWithSize(2));
         assertThat(
@@ -148,8 +148,8 @@ public void testInstrumentationTargetLookupWithOverloads() throws IOException, C
         );
     }
 
-    public void testInstrumentationTargetLookupWithCtors() throws IOException, ClassNotFoundException {
-        Map<MethodKey, CheckMethod> checkMethods = instrumentationService.lookupMethodsToInstrument(TestCheckerCtors.class.getName());
+    public void testInstrumentationTargetLookupWithCtors() throws IOException {
+        Map<MethodKey, CheckMethod> checkMethods = instrumentationService.lookupMethods(TestCheckerCtors.class);
 
         assertThat(checkMethods, aMapWithSize(2));
         assertThat(