Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…esql-kql-function
afoucret · Nov 22, 2024 · 8bd0239 · 8bd0239
2 parents 02ea75e + f922353
commit 8bd0239
Show file tree

Hide file tree

Showing 364 changed files with 23,541 additions and 5,801 deletions.
diff --git a/.backportrc.json b/.backportrc.json
@@ -1,10 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.17.0$" : "8.x",
+    "^v8.18.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.1", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml
@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -9,6 +9,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -288,8 +289,8 @@ steps:
         env:
           BWC_VERSION: 8.15.4
 
-      - label: "{{matrix.image}} / 8.16.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.1
+      - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -302,7 +303,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.1
+          BWC_VERSION: 8.16.2
 
       - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
@@ -320,6 +321,22 @@ steps:
         env:
           BWC_VERSION: 8.17.0
 
+      - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.18.0
+
       - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
         timeout_in_minutes: 300

diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml
@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.1#bwcTest
+      - label: 8.16.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.1
+          BWC_VERSION: 8.16.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -344,6 +344,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.18.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.18.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
         timeout_in_minutes: 300
@@ -429,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.1", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -471,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.1", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml
@@ -11,6 +11,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -38,6 +39,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -65,6 +67,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -15,6 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.4"
-  - "8.16.1"
+  - "8.16.2"
   - "8.17.0"
+  - "8.18.0"
   - "9.0.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,4 +1,5 @@
 BWC_VERSION:
-  - "8.16.1"
+  - "8.16.2"
   - "8.17.0"
+  - "8.18.0"
   - "9.0.0"
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.plugins.PluginsLoader;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.DocReader;
@@ -76,8 +77,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        null,
-        Path.of(System.getProperty("plugins.dir"))
+        new PluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

diff --git a/branches.json b/branches.json
@@ -7,6 +7,9 @@
     {
       "branch": "8.16"
     },
+    {
+      "branch": "8.17"
+    },
     {
       "branch": "8.x"
     },

diff --git a/...l/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy b/...l/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.gradle.internal
 
-
 import spock.lang.Unroll
 import com.github.tomakehurst.wiremock.WireMockServer
 
@@ -24,22 +23,19 @@ import java.nio.file.Paths
 import java.util.regex.Matcher
 import java.util.regex.Pattern
 
-import static org.elasticsearch.gradle.internal.JdkDownloadPlugin.VENDOR_ADOPTIUM
-import static org.elasticsearch.gradle.internal.JdkDownloadPlugin.VENDOR_OPENJDK
+import static org.elasticsearch.gradle.internal.JdkDownloadPlugin.*
 
 class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
     private static final String OPENJDK_VERSION_OLD = "1+99"
     private static final String ADOPT_JDK_VERSION = "12.0.2+10"
     private static final String ADOPT_JDK_VERSION_11 = "11.0.10+9"
     private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
+    private static final String AZUL_JDK_VERSION_8 = "8u302+b08"
+    private static final String AZUL_8_DISTRO_VERSION = "8.56.0.23"
     private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
     private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)")
 
-    def setup() {
-        configurationCacheCompatible = false
-    }
-
     @Unroll
     def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
         given:
@@ -56,14 +52,16 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
                 version = '$jdkVersion'
                 platform = "$platform"
                 architecture = '$arch'
+                distributionVersion = '$distributionVersion'
               }
             }
 
-            def theJdks = jdks
+//            def theJdks = jdks
             tasks.register("getJdk") {
                 dependsOn jdks.myJdk
+                def jdk = jdks.myJdk
                 doLast {
-                    println "JDK HOME: " + theJdks.myJdk
+                    println "JDK HOME: " + jdk
                 }
             }
         """
@@ -78,22 +76,23 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         assertExtraction(result.output, expectedJavaBin);
 
         where:
-        platform  | arch      | jdkVendor           | jdkVersion           | expectedJavaBin          | suffix
-        "linux"   | "x64"     | VENDOR_ADOPTIUM     | ADOPT_JDK_VERSION    | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "bin/java"               | "(old version)"
-        "windows" | "x64"     | VENDOR_ADOPTIUM     | ADOPT_JDK_VERSION    | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "bin/java"               | "(old version)"
-        "darwin"  | "x64"     | VENDOR_ADOPTIUM     | ADOPT_JDK_VERSION    | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "Contents/Home/bin/java" | "(old version)"
-        "mac"     | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "Contents/Home/bin/java" | ""
-        "mac"     | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "Contents/Home/bin/java" | "(old version)"
-        "darwin"  | "aarch64" | VENDOR_ADOPTIUM     | ADOPT_JDK_VERSION    | "Contents/Home/bin/java" | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM     | ADOPT_JDK_VERSION    | "bin/java"               | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM     | ADOPT_JDK_VERSION_11 | "bin/java"               | "(jdk 11)"
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM     | ADOPT_JDK_VERSION_15 | "bin/java"               | "(jdk 15)"
+        platform  | arch      | jdkVendor       | jdkVersion           | distributionVersion   | expectedJavaBin          | suffix
+        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
+        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
+        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
+        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | null                  | "bin/java"               | "(jdk 11)"
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | null                  | "bin/java"               | "(jdk 15)"
+        "darwin"  | "aarch64" | VENDOR_ZULU     | AZUL_JDK_VERSION_8   | AZUL_8_DISTRO_VERSION | "Contents/Home/bin/java" | "(jdk 8)"
     }
 
     def "transforms are reused across projects"() {
@@ -118,9 +117,10 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
                   }
                 }
                 tasks.register("getJdk") {
-                    dependsOn jdks.myJdk
+                    def jdk = jdks.myJdk
+                    dependsOn jdk
                     doLast {
-                        println "JDK HOME: " + jdks.myJdk
+                        println "JDK HOME: " + jdk
                     }
                 }
             """
@@ -137,7 +137,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         result.output.count("Unpacking linux-12.0.2-x64.tar.gz using ${SymbolicLinkPreservingUntarTransform.simpleName}") == 1
 
         where:
-        platform | jdkVendor           | jdkVersion        | expectedJavaBin
+        platform | jdkVendor       | jdkVersion        | expectedJavaBin
         "linux"  | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "bin/java"
     }
 
@@ -159,6 +159,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
                 vendor = '$VENDOR_ADOPTIUM'
                 version = '$ADOPT_JDK_VERSION'
                 platform = "$platform"
+                distributionVersion = '$ADOPT_JDK_VERSION'
                 architecture = "x64"
               }
             }
@@ -204,6 +205,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         assert matcher.find() == true;
         String jdkHome = matcher.group(1);
         Path javaPath = Paths.get(jdkHome, javaBin);
+        println "canonical " + javaPath.toFile().getCanonicalPath()
+        Paths.get(jdkHome).toFile().listFiles().each { println it }
         assert Files.exists(javaPath) == true;
         true
     }
@@ -221,15 +224,26 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
             final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
             final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
             return "/java/GA/" + versionPath + "/GPL/" + filename;
+        } else if (vendor.equals(VENDOR_ZULU)) {
+            // we only have a single version of zulu currently in the tests
+            return "/zulu/bin/zulu8.56.0.23-ca-jdk8.0.302-macosx_aarch64.tar.gz"
         }
     }
 
     private static byte[] filebytes(final String vendor, final String platform) throws IOException {
         final String effectivePlatform = getPlatform(vendor, platform);
         if (vendor.equals(VENDOR_ADOPTIUM)) {
-            return JdkDownloadPluginFuncTest.class.getResourceAsStream("fake_adoptium_" + effectivePlatform + "." + extension(platform)).getBytes()
+            return JdkDownloadPluginFuncTest.class.getResourceAsStream(
+                "fake_adoptium_" + effectivePlatform + "." + extension(platform)
+            ).getBytes()
         } else if (vendor.equals(VENDOR_OPENJDK)) {
-            JdkDownloadPluginFuncTest.class.getResourceAsStream("fake_openjdk_" + effectivePlatform + "." + extension(platform)).getBytes()
+            return JdkDownloadPluginFuncTest.class.getResourceAsStream(
+                "fake_openjdk_" + effectivePlatform + "." + extension(platform)
+            ).getBytes()
+        } else {
+            // zulu
+            String resourcePath = "fake_zulu_" + effectivePlatform + "." + extension(platform)
+            return JdkDownloadPluginFuncTest.class.getResourceAsStream(resourcePath).getBytes()
         }
     }
 

diff --git a/...internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_zulu_macos.tar.gz b/...internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_zulu_macos.tar.gz
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle
@@ -25,12 +25,12 @@ if (buildParams.inFipsJvm) {
     File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename)
     File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy')
     File fipsTrustStore = new File(fipsResourcesDir, 'cacerts.bcfks')
-    def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.4')
-    def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.17')
+    def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.5')
+    def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19')
     def manualDebug = false; //change this to manually debug bouncy castle in an IDE
     if(manualDebug) {
-      bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.4')
-      bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.17'){
+      bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.5')
+      bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19'){
         exclude group: 'org.bouncycastle', module: 'bc-fips'  // to avoid jar hell
       }
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,6 +7,9 @@ @@
         {
           "branch": "8.16"
         },
+        {
+          "branch": "8.17"
+        },
         {
           "branch": "8.x"
         },
@@ Expand Down @@