Merge branch 'main' into profiling-arrays

danielmitterdorfer · Feb 7, 2024 · 8c1ca50 · 8c1ca50
2 parents 72a20e7 + 25dd12d
commit 8c1ca50
Show file tree

Hide file tree

Showing 367 changed files with 11,295 additions and 4,341 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.18", "8.12.1", "8.13.0"]
+            BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -1121,6 +1121,22 @@ steps:
         env:
           BWC_VERSION: 7.17.18
 
+      - label: "{{matrix.image}} / 7.17.19 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.19
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.19
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1841,6 +1857,22 @@ steps:
         env:
           BWC_VERSION: 8.12.1
 
+      - label: "{{matrix.image}} / 8.12.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.2
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.2
+
       - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
         timeout_in_minutes: 300

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -682,6 +682,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.18
+      - label: 7.17.19 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.19#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.19
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1132,6 +1142,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.1
+      - label: 8.12.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.2#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.2
       - label: 8.13.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
         timeout_in_minutes: 300

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -67,6 +67,7 @@ BWC_VERSION:
   - "7.17.16"
   - "7.17.17"
   - "7.17.18"
+  - "7.17.19"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -112,4 +113,5 @@ BWC_VERSION:
   - "8.11.4"
   - "8.12.0"
   - "8.12.1"
+  - "8.12.2"
   - "8.13.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.18"
-  - "8.12.1"
+  - "7.17.19"
+  - "8.12.2"
   - "8.13.0"
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -114,7 +114,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Builds artifacts needed as dependency for IDE modules'
     dependsOn([':client:rest-high-level:shadowJar',
       ':plugins:repository-hdfs:hadoop-client-api:shadowJar',
-      ':libs:elasticsearch-x-content:generateProviderImpl',
+      ':libs:elasticsearch-x-content:generateImplProviderImpl',
       ':x-pack:plugin:esql:compute:ann:jar',
       ':x-pack:plugin:esql:compute:gen:jar',
       ':server:generateModulesList',

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
@@ -11,11 +11,14 @@
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.JavaCompile;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.jvm.tasks.Jar;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -72,9 +75,19 @@ private void addMrjarSourceset(Project project, JavaPluginExtension javaExtensio
         SourceSet sourceSet = javaExtension.getSourceSets().maybeCreate(sourcesetName);
         GradleUtils.extendSourceSet(project, SourceSet.MAIN_SOURCE_SET_NAME, sourcesetName);
 
-        project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME).configure(jarTask -> {
-            jarTask.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput()));
-            jarTask.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); });
+        var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
+        jarTask.configure(task -> {
+            task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput()));
+            task.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); });
+        });
+
+        project.getTasks().withType(Test.class).named(JavaPlugin.TEST_TASK_NAME).configure(testTask -> {
+            testTask.dependsOn(jarTask);
+
+            SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(project);
+            FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
+            FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
+            testTask.setClasspath(testRuntime.minus(mainRuntime).plus(project.files(jarTask)));
         });
 
         project.getTasks().withType(JavaCompile.class).named(sourceSet.getCompileJavaTaskName()).configure(compileTask -> {

diff --git a/docs/changelog/102885.yaml b/docs/changelog/102885.yaml
@@ -0,0 +1,5 @@
+pr: 102885
+summary: Make field limit more predictable
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/103511.yaml b/docs/changelog/103511.yaml
diff --git a/docs/changelog/103741.yaml b/docs/changelog/103741.yaml
diff --git a/docs/changelog/103817.yaml b/docs/changelog/103817.yaml
diff --git a/docs/changelog/103819.yaml b/docs/changelog/103819.yaml
diff --git a/docs/changelog/104087.yaml b/docs/changelog/104087.yaml
diff --git a/docs/changelog/104145.yaml b/docs/changelog/104145.yaml
diff --git a/docs/changelog/104198.yaml b/docs/changelog/104198.yaml
diff --git a/docs/changelog/104281.yaml b/docs/changelog/104281.yaml
diff --git a/docs/changelog/104288.yaml b/docs/changelog/104288.yaml
diff --git a/docs/changelog/104289.yaml b/docs/changelog/104289.yaml
diff --git a/docs/changelog/104314.yaml b/docs/changelog/104314.yaml
diff --git a/docs/changelog/104418.yaml b/docs/changelog/104418.yaml
diff --git a/docs/changelog/104440.yaml b/docs/changelog/104440.yaml
@@ -0,0 +1,6 @@
+pr: 104440
+summary: Fix write index resolution when an alias is pointing to a TSDS
+area: Data streams
+type: bug
+issues:
+  - 104189
diff --git a/docs/changelog/104483.yaml b/docs/changelog/104483.yaml
@@ -0,0 +1,5 @@
+pr: 104483
+summary: Make `task_type` optional in `_inference` APIs
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/104523.yaml b/docs/changelog/104523.yaml
diff --git a/docs/changelog/104585.yaml b/docs/changelog/104585.yaml
diff --git a/docs/changelog/104586.yaml b/docs/changelog/104586.yaml
diff --git a/docs/changelog/104591.yaml b/docs/changelog/104591.yaml
diff --git a/docs/changelog/104600.yaml b/docs/changelog/104600.yaml
diff --git a/docs/changelog/104606.yaml b/docs/changelog/104606.yaml
diff --git a/docs/changelog/104722.yaml b/docs/changelog/104722.yaml
diff --git a/docs/changelog/104802.yaml b/docs/changelog/104802.yaml
diff --git a/docs/changelog/104808.yaml b/docs/changelog/104808.yaml
diff --git a/docs/changelog/104832.yaml b/docs/changelog/104832.yaml
diff --git a/docs/changelog/104891.yaml b/docs/changelog/104891.yaml
diff --git a/docs/changelog/104904.yaml b/docs/changelog/104904.yaml
diff --git a/docs/changelog/104905.yaml b/docs/changelog/104905.yaml
@@ -0,0 +1,6 @@
+pr: 104905
+summary: "Execute lazy rollover with an internal dedicated user #104732"
+area: Data streams
+type: bug
+issues:
+ - 104732
diff --git a/docs/changelog/104949.yaml b/docs/changelog/104949.yaml
@@ -0,0 +1,5 @@
+pr: 104949
+summary: Add text_embedding inference service with multilingual-e5 and custom eland models
+area: Machine Learning
+type: enhancement
+issues: [ ]
diff --git a/docs/changelog/105048.yaml b/docs/changelog/105048.yaml
@@ -0,0 +1,6 @@
+pr: 105048
+summary: "ES|QL: Fix exception handling on `date_parse` with wrong date pattern"
+area: ES|QL
+type: bug
+issues:
+ - 104124
diff --git a/docs/changelog/105061.yaml b/docs/changelog/105061.yaml
@@ -0,0 +1,6 @@
+pr: 105061
+summary: "ESQL: Push CIDR_MATCH to Lucene if possible"
+area: ES|QL
+type: bug
+issues:
+ - 105042
diff --git a/docs/changelog/105064.yaml b/docs/changelog/105064.yaml
@@ -0,0 +1,17 @@
+pr: 105064
+summary: "ES|QL: remove PROJECT keyword from the grammar"
+area: ES|QL
+type: breaking
+issues: []
+breaking:
+  title: "ES|QL: remove PROJECT keyword from the grammar"
+  area: REST API
+  details: "Removes the PROJECT keyword (an alias for KEEP) from ES|QL grammar"
+  impact: "Before this change, users could use PROJECT as an alias for KEEP in ESQL queries,\
+    \ (eg. 'FROM idx | PROJECT name, surname')\
+    \ the parser replaced PROJECT with KEEP, emitted a warning:\
+    \ 'PROJECT command is no longer supported, please use KEEP instead'\
+    \ and the query was executed normally.\
+    \ With this change, PROJECT command is no longer recognized by the query parser;\
+    \ queries using PROJECT command now return a parsing exception."
+  notable: false
diff --git a/docs/changelog/105081.yaml b/docs/changelog/105081.yaml
@@ -0,0 +1,6 @@
+pr: 105081
+summary: For empty mappings use a `LocalRelation`
+area: ES|QL
+type: bug
+issues:
+ - 104809
diff --git a/docs/changelog/105096.yaml b/docs/changelog/105096.yaml
@@ -0,0 +1,5 @@
+pr: 105096
+summary: Harden index mapping parameter check in enrich runner
+area: Ingest Node
+type: bug
+issues: []