Indices options group wildcards

.buildkite/scripts/fixture-deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin docker.elastic.co
+unset DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD
+
+docker buildx create --use
+.ci/scripts/run-gradle.sh deployFixtureDockerImages

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationPrecommitPlugin.java

@@ -30,11 +30,11 @@ public TaskProvider<? extends Task> createTask(Project project) {
                 .register("validate" + publicationName + "Pom", PomValidationTask.class);
             validatePom.configure(t -> t.dependsOn(validateTask));
             validateTask.configure(task -> {
-                GenerateMavenPom generateMavenPom = project.getTasks()
+                TaskProvider<GenerateMavenPom> generateMavenPom = project.getTasks()
                     .withType(GenerateMavenPom.class)
-                    .getByName("generatePomFileFor" + publicationName + "Publication");
+                    .named("generatePomFileFor" + publicationName + "Publication");
                 task.dependsOn(generateMavenPom);
-                task.getPomFile().fileValue(generateMavenPom.getDestination());
+                task.getPomFile().fileProvider(generateMavenPom.map(GenerateMavenPom::getDestination));
             });
         });
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java

@@ -20,8 +20,10 @@
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.SetProperty;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.PathSensitive;
 import org.gradle.api.tasks.PathSensitivity;
@@ -36,14 +38,15 @@
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
 /**
  * This task wraps up the details of building a Docker image, including adding a pull
  * mechanism that can retry, and emitting the image SHA as a task output.
  */
-public class DockerBuildTask extends DefaultTask {
+public abstract class DockerBuildTask extends DefaultTask {
     private static final Logger LOGGER = Logging.getLogger(DockerBuildTask.class);
 
     private final WorkerExecutor workerExecutor;
@@ -55,15 +58,13 @@ public class DockerBuildTask extends DefaultTask {
     private boolean noCache = true;
     private String[] baseImages;
     private MapProperty<String, String> buildArgs;
-    private Property<String> platform;
 
     @Inject
     public DockerBuildTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory, ProjectLayout projectLayout) {
         this.workerExecutor = workerExecutor;
         this.markerFile = objectFactory.fileProperty();
         this.dockerContext = objectFactory.directoryProperty();
         this.buildArgs = objectFactory.mapProperty(String.class, String.class);
-        this.platform = objectFactory.property(String.class).convention(Architecture.current().dockerPlatform);
         this.markerFile.set(projectLayout.getBuildDirectory().file("markers/" + this.getName() + ".marker"));
     }
 
@@ -75,9 +76,10 @@ public void build() {
             params.getTags().set(Arrays.asList(tags));
             params.getPull().set(pull);
             params.getNoCache().set(noCache);
+            params.getPush().set(getPush().getOrElse(false));
             params.getBaseImages().set(Arrays.asList(baseImages));
             params.getBuildArgs().set(buildArgs);
-            params.getPlatform().set(platform);
+            params.getPlatforms().set(getPlatforms());
         });
     }
 
@@ -129,10 +131,16 @@ public MapProperty<String, String> getBuildArgs() {
     }
 
     @Input
-    public Property<String> getPlatform() {
-        return platform;
+    public abstract SetProperty<String> getPlatforms();
+
+    public void setPlatform(String platform) {
+        getPlatforms().set(Arrays.asList(platform));
     }
 
+    @Input
+    @Optional
+    public abstract Property<Boolean> getPush();
+
     @OutputFile
     public RegularFileProperty getMarkerFile() {
         return markerFile;
@@ -181,7 +189,7 @@ public void execute() {
             }
 
             final List<String> tags = parameters.getTags().get();
-            final boolean isCrossPlatform = parameters.getPlatform().get().equals(Architecture.current().dockerPlatform) == false;
+            final boolean isCrossPlatform = isCrossPlatform();
 
             LoggedExec.exec(execOperations, spec -> {
                 spec.executable("docker");
@@ -193,7 +201,7 @@ public void execute() {
                 spec.args("build", parameters.getDockerContext().get().getAsFile().getAbsolutePath());
 
                 if (isCrossPlatform) {
-                    spec.args("--platform", parameters.getPlatform().get());
+                    spec.args("--platform", parameters.getPlatforms().get().stream().collect(Collectors.joining(",")));
                 }
 
                 if (parameters.getNoCache().get()) {
@@ -203,18 +211,34 @@ public void execute() {
                 tags.forEach(tag -> spec.args("--tag", tag));
 
                 parameters.getBuildArgs().get().forEach((k, v) -> spec.args("--build-arg", k + "=" + v));
+
+                if (parameters.getPush().getOrElse(false)) {
+                    spec.args("--push");
+                }
             });
 
             // Fetch the Docker image's hash, and write it to desk as the task's output. Doing this allows us
             // to do proper up-to-date checks in Gradle.
             try {
+                // multi-platform image builds do not end up in local registry, so we need to pull the just build image
+                // first to get the checksum and also serves as a test for the image being pushed correctly
+                if (parameters.getPlatforms().get().size() > 1 && parameters.getPush().getOrElse(false)) {
+                    pullBaseImage(tags.get(0));
+                }
                 final String checksum = getImageChecksum(tags.get(0));
                 Files.writeString(parameters.getMarkerFile().getAsFile().get().toPath(), checksum + "\n");
             } catch (IOException e) {
                 throw new RuntimeException("Failed to write marker file", e);
             }
         }
 
+        private boolean isCrossPlatform() {
+            return getParameters().getPlatforms()
+                .get()
+                .stream()
+                .anyMatch(any -> any.equals(Architecture.current().dockerPlatform) == false);
+        }
+
         private String getImageChecksum(String imageTag) {
             final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
 
@@ -243,6 +267,8 @@ interface Parameters extends WorkParameters {
 
         MapProperty<String, String> getBuildArgs();
 
-        Property<String> getPlatform();
+        SetProperty<String> getPlatforms();
+
+        Property<Boolean> getPush();
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java

@@ -28,6 +28,7 @@
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -187,7 +188,7 @@ public void checkDependencies() {
         }
         File licensesDirAsFile = licensesDir.get().getAsFile();
         if (dependencies.isEmpty()) {
-            if (licensesDirAsFile.exists()) {
+            if (licensesDirAsFile.exists() && allIgnored() == false) {
                 throw new GradleException("Licenses dir " + licensesDirAsFile + " exists, but there are no dependencies");
             }
             return; // no dependencies to check
@@ -227,6 +228,10 @@ public void checkDependencies() {
         sources.forEach((item, exists) -> failIfAnyMissing(item, exists, "sources"));
     }
 
+    private boolean allIgnored() {
+        return Arrays.asList(getLicensesDir().listFiles()).stream().map(f -> f.getName()).allMatch(ignoreFiles::contains);
+    }
+
     // This is just a marker output folder to allow this task being up-to-date.
     // The check logic is exception driven so a failed tasks will not be defined
     // by this output but when successful we can safely mark the task as up-to-date.

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java

@@ -63,8 +63,17 @@ public void execute(DependencyLicensesTask dependencyLicensesTask) {
     public void givenProjectWithLicensesDirButNoDependenciesThenShouldThrowException() throws Exception {
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("exists, but there are no dependencies"));
+        getLicensesDir(project).mkdir();
+        createFileIn(getLicensesDir(project), "groovy-LICENSE.txt", PERMISSIVE_LICENSE_TEXT);
+        task.get().checkDependencies();
+    }
 
+    @Test
+    public void givenProjectWithLicensesDirButAllIgnoreFileAndNoDependencies() throws Exception {
         getLicensesDir(project).mkdir();
+        String licenseFileName = "cloudcarbonfootprint-LICENSE.txt";
+        createFileIn(getLicensesDir(project), licenseFileName, PERMISSIVE_LICENSE_TEXT);
+        task.get().ignoreFile(licenseFileName);
         task.get().checkDependencies();
     }
 

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java

@@ -34,8 +34,8 @@
 
 public class NoopPlugin extends Plugin implements ActionPlugin {
 
-    public static final ActionType<SearchResponse> NOOP_SEARCH_ACTION = new ActionType<>("mock:data/read/search", SearchResponse::new);
-    public static final ActionType<BulkResponse> NOOP_BULK_ACTION = new ActionType<>("mock:data/write/bulk", BulkResponse::new);
+    public static final ActionType<SearchResponse> NOOP_SEARCH_ACTION = new ActionType<>("mock:data/read/search");
+    public static final ActionType<BulkResponse> NOOP_BULK_ACTION = new ActionType<>("mock:data/write/bulk");
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {

distribution/docker/build.gradle

@@ -398,7 +398,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
 
       noCache = BuildParams.isCi
       tags = generateTags(base, architecture)
-      platform = architecture.dockerPlatform
+      platforms.add(architecture.dockerPlatform)
 
       // We don't build the Iron Bank image when we release Elasticsearch, as there's
       // separate process for submitting new releases. However, for testing we do a
@@ -468,7 +468,7 @@ void addBuildEssDockerImageTask(Architecture architecture) {
       noCache = BuildParams.isCi
       baseImages = []
       tags = generateTags(base, architecture)
-      platform = architecture.dockerPlatform
+      platforms.add(architecture.dockerPlatform)
 
       onlyIf("$architecture supported") { isArchitectureSupported(architecture) }
     }

docs/changelog/103656.yaml

@@ -0,0 +1,5 @@
+pr: 103656
+summary: "ESQL: add =~ operator (case insensitive equality)"
+area: ES|QL
+type: feature
+issues: []

docs/changelog/104559.yaml

@@ -0,0 +1,5 @@
+pr: 104559
+summary: Adding support for Cohere inference service
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/104591.yaml

@@ -0,0 +1,5 @@
+pr: 104591
+summary: Avoid execute ESQL planning on refresh thread
+area: ES|QL
+type: bug
+issues: []

docs/changelog/104643.yaml

@@ -0,0 +1,5 @@
+pr: 104643
+summary: "[Connectors API] Implement update service type action"
+area: Application
+type: enhancement
+issues: []

docs/changelog/104718.yaml

@@ -0,0 +1,6 @@
+pr: 104718
+summary: "ESQL: Fix replacement of nested expressions in aggs with multiple parameters"
+area: ES|QL
+type: bug
+issues:
+ - 104706

docs/changelog/104722.yaml

@@ -0,0 +1,6 @@
+pr: 104722
+summary: Avoid possible datafeed infinite loop with filtering aggregations
+area: Machine Learning
+type: bug
+issues:
+ - 104699

docs/reference/esql/functions/asin.asciidoc

@@ -14,9 +14,8 @@ Numeric expression. If `null`, the function returns `null`.
 
 *Description*
 
-Returns the
-https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[arcsine]
-of the input numeric expression as an angle, expressed in radians.
+Returns the {wikipedia}/Inverse_trigonometric_functions[arcsine] of the input
+numeric expression as an angle, expressed in radians.
 
 *Supported types*
 

docs/reference/esql/functions/atan.asciidoc

@@ -14,9 +14,8 @@ Numeric expression. If `null`, the function returns `null`.
 
 *Description*
 
-Returns the
-https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[arctangent] of the
-input numeric expression as an angle, expressed in radians.
+Returns the {wikipedia}/Inverse_trigonometric_functions[arctangent] of the input
+numeric expression as an angle, expressed in radians.
 
 *Supported types*
 

docs/reference/esql/functions/atan2.asciidoc

@@ -17,9 +17,8 @@ Numeric expression. If `null`, the function returns `null`.
 
 *Description*
 
-The https://en.wikipedia.org/wiki/Atan2[angle] between the positive x-axis and
-the ray from the origin to the point (x , y) in the Cartesian plane, expressed
-in radians.
+The {wikipedia}/Atan2[angle] between the positive x-axis and the ray from the
+origin to the point (x , y) in the Cartesian plane, expressed in radians.
 
 *Supported types*
 

docs/reference/esql/functions/auto_bucket.asciidoc

@@ -6,13 +6,13 @@
 
 [source,esql]
 ----
-AUTO_BUCKET(field, buckets, from, to)
+AUTO_BUCKET(expression, buckets, from, to)
 ----
 
 *Parameters*
 
 `field`::
-Numeric or date column from which to derive buckets.
+Numeric or date expression from which to derive buckets.
 
 `buckets`::
 Target number of buckets.

docs/reference/esql/functions/avg.asciidoc

@@ -6,15 +6,15 @@
 
 [source,esql]
 ----
-AVG(column)
+AVG(expression)
 ----
 
-`column`::
-Numeric column. If `null`, the function returns `null`.
+`expression`::
+Numeric expression. If `null`, the function returns `null`.
 
 *Description*
 
-The average of a numeric field.
+The average of a numeric expression.
 
 *Supported types*
 

docs/reference/esql/functions/cos.asciidoc

@@ -14,8 +14,8 @@ Numeric expression. If `null`, the function returns `null`.
 
 *Description*
 
-Returns the https://en.wikipedia.org/wiki/Sine_and_cosine[cosine] of `n`. Input
-expected in radians.
+Returns the {wikipedia}/Sine_and_cosine[cosine] of `n`. Input expected in
+radians.
 
 *Supported types*
 

docs/reference/esql/functions/cosh.asciidoc

@@ -12,14 +12,13 @@ image::esql/functions/signature/cosh.svg[Embedded,opts=inline]
 `n`::
 Numeric expression. If `null`, the function returns `null`.
 
-*Supported types*
+*Description*
 
-include::types/cosh.asciidoc[]
+Returns the {wikipedia}/Hyperbolic_functions[hyperbolic cosine].
 
-*Description*
+*Supported types*
 
-Returns the https://en.wikipedia.org/wiki/Hyperbolic_functions[hyperbolic
-cosine].
+include::types/cosh.asciidoc[]
 
 *Example*