Merge branch 'main' into esql-translator-prototype-3

not-napoleon · Mar 12, 2024 · 434a34f · 434a34f
2 parents f37cfae + bfbb155
commit 434a34f
Show file tree

Hide file tree

Showing 1,184 changed files with 32,995 additions and 11,561 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0", "8.14.0"]
+            BWC_VERSION: ["7.17.19", "8.12.3", "8.13.0", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -1873,6 +1873,22 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
+      - label: "{{matrix.image}} / 8.12.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.3
+
       - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
         timeout_in_minutes: 300

diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml
@@ -49,7 +49,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: gcp
@@ -59,6 +58,22 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-fips-matrix-bwc"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - openjdk17
+            BWC_VERSION: $BWC_LIST
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - group: java-matrix
     steps:
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
@@ -69,9 +84,6 @@ steps:
             ES_RUNTIME_JAVA:
               - graalvm-ce17
               - openjdk17
-              - openjdk18
-              - openjdk19
-              - openjdk20
               - openjdk21
               - openjdk22
             GRADLE_TASK:
@@ -88,6 +100,25 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-matrix-bwc"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - graalvm-ce17
+              - openjdk17
+              - openjdk21
+              - openjdk22
+            BWC_VERSION: $BWC_LIST
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
     timeout_in_minutes: 360

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -1152,6 +1152,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.2
+      - label: 8.12.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.3
       - label: 8.13.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
         timeout_in_minutes: 300
@@ -1220,7 +1230,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: gcp
@@ -1230,6 +1239,22 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-fips-matrix-bwc"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - openjdk17
+            BWC_VERSION: ["7.17.19", "8.12.3", "8.13.0", "8.14.0"]
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - group: java-matrix
     steps:
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
@@ -1240,9 +1265,6 @@ steps:
             ES_RUNTIME_JAVA:
               - graalvm-ce17
               - openjdk17
-              - openjdk18
-              - openjdk19
-              - openjdk20
               - openjdk21
               - openjdk22
             GRADLE_TASK:
@@ -1259,6 +1281,25 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-matrix-bwc"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - graalvm-ce17
+              - openjdk17
+              - openjdk21
+              - openjdk22
+            BWC_VERSION: ["7.17.19", "8.12.3", "8.13.0", "8.14.0"]
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
     timeout_in_minutes: 360

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -114,5 +114,6 @@ BWC_VERSION:
   - "8.12.0"
   - "8.12.1"
   - "8.12.2"
+  - "8.12.3"
   - "8.13.0"
   - "8.14.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.19"
-  - "8.12.2"
+  - "8.12.3"
   - "8.13.0"
   - "8.14.0"
diff --git a/README.asciidoc b/README.asciidoc
@@ -1,20 +1,24 @@
 = Elasticsearch
 
-Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. You can use Elasticsearch to perform real-time search over massive datasets for applications including:
+Elasticsearch is a distributed search and analytics engine optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more.
 
-* Vector search
+Use cases enabled by Elasticsearch include:
+
+* https://www.elastic.co/search-labs/blog/articles/retrieval-augmented-generation-rag[Retrieval Augmented Generation (RAG)] 
+* https://www.elastic.co/search-labs/blog/categories/vector-search[Vector search]
 * Full-text search
 * Logs
 * Metrics
 * Application performance monitoring (APM)
 * Security logs
 
-
 \... and more!
 
 To learn more about Elasticsearch's features and capabilities, see our
 https://www.elastic.co/products/elasticsearch[product page].
 
+To access information on https://www.elastic.co/search-labs/blog/categories/ml-research[machine learning innovations] and the latest https://www.elastic.co/search-labs/blog/categories/lucene[Lucene contributions from Elastic], more information can be found in https://www.elastic.co/search-labs[Search Labs]. 
+
 [[get-started]]
 == Get started
 

diff --git a/...l/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy b/...l/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy
@@ -99,17 +99,17 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         given:
         def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform)
         def mockedContent = filebytes(jdkVendor, platform)
-        3.times {
-            settingsFile << """
-                include ':sub-$it'
-            """
-        }
         buildFile.text = """
             plugins {
              id 'elasticsearch.jdk-download' apply false
             }
 
             subprojects {
+
+            }
+        """
+        3.times {
+            subProject(':sub-' + it) << """
                 apply plugin: 'elasticsearch.jdk-download'
 
                 jdks {
@@ -126,8 +126,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
                         println "JDK HOME: " + jdks.myJdk
                     }
                 }
-            }
-        """
+            """
+        }
 
         when:
         def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
@@ -165,7 +165,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
                 architecture = "x64"
               }
             }
-            
+
             tasks.register("getJdk", PrintJdk) {
                 dependsOn jdks.myJdk
                 jdkPath = jdks.myJdk.getPath()
@@ -174,7 +174,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
             class PrintJdk extends DefaultTask {
                 @Input
                 String jdkPath
-                
+
                 @TaskAction void print() {
                     println "JDK HOME: " + jdkPath
                 }

diff --git a/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -32,7 +32,9 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 
 import javax.inject.Inject;
 
@@ -142,6 +144,8 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
 
                     System.out.println("Generating buildscan link for artifact...");
 
+                    // Output should be in the format: "<UUID><space><ISO-8601-timestamp>\n"
+                    // and multiple artifacts could be returned
                     Process process = new ProcessBuilder(
                         "buildkite-agent",
                         "artifact",
@@ -150,7 +154,7 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                         "--step",
                         System.getenv("BUILDKITE_JOB_ID"),
                         "--format",
-                        "%i"
+                        "%i %c"
                     ).start();
                     process.waitFor();
                     String processOutput;
@@ -159,7 +163,17 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                     } catch (IOException e) {
                         processOutput = "";
                     }
-                    String artifactUuid = processOutput.trim();
+
+                    // Sort them by timestamp, and grab the most recent one
+                    Optional<String> artifact = Arrays.stream(processOutput.trim().split("\n")).map(String::trim).min((a, b) -> {
+                        String[] partsA = a.split(" ");
+                        String[] partsB = b.split(" ");
+                        // ISO-8601 timestamps can be sorted lexicographically
+                        return partsB[1].compareTo(partsA[1]);
+                    });
+
+                    // Grab just the UUID from the artifact
+                    String artifactUuid = artifact.orElse("").split(" ")[0];
 
                     System.out.println("Artifact UUID: " + artifactUuid);
                     if (artifactUuid.isEmpty() == false) {

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.14.0
-lucene            = 9.9.2
+lucene            = 9.10.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac
@@ -13,7 +13,7 @@ supercsv          = 2.4.0
 log4j             = 2.19.0
 slf4j             = 2.0.6
 ecsLogging        = 1.2.0
-jna               = 5.10.0
+jna               = 5.12.1
 netty             = 4.1.107.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1

diff --git a/...s/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy b/...s/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy
@@ -70,25 +70,14 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
         def version = VersionProperties.getElasticsearch()
         def platform = ElasticsearchDistribution.Platform.LINUX
 
-        3.times {
-            settingsFile << """
-                include ':sub-$it'
-            """
-        }
         buildFile.text = """
-            import org.elasticsearch.gradle.Architecture
-
             plugins {
                 id 'elasticsearch.distribution-download'
             }
-
-            subprojects {
-                apply plugin: 'elasticsearch.distribution-download'
-
-                ${setupTestDistro(version, platform)}
-                ${setupDistroTask()}
-            }
         """
+        3.times {
+            subProject(':sub-' + it) << applyPluginAndSetupDistro(version, platform)
+        }
 
         when:
         def runner = gradleRunner('setupDistro', '-i', '-g', gradleUserHome)
@@ -118,14 +107,6 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
                 id 'elasticsearch.distribution-download'
             }
 
-            ${setupTestDistro(version, platform)}
-            ${setupDistroTask()}
-
-        """
-    }
-
-    private static String setupTestDistro(String version, ElasticsearchDistribution.Platform platform) {
-        return """
             elasticsearch_distributions {
                 test_distro {
                     version = "$version"
@@ -134,11 +115,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
                     architecture = Architecture.current();
                 }
             }
-            """
-    }
 
-    private static String setupDistroTask() {
-        return """
             tasks.register("setupDistro", Sync) {
                 from(elasticsearch_distributions.test_distro)
                 into("build/distro")