Merge branch 'main' into functioninfo

fang-xing-esql · Mar 19, 2024 · f2530d9 · f2530d9
2 parents be168ba + d5565b6
commit f2530d9
Show file tree

Hide file tree

Showing 415 changed files with 3,522 additions and 774 deletions.
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.benchmark.compute.operator;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
@@ -26,11 +27,13 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.type.EsField;
@@ -58,7 +61,6 @@
 @State(Scope.Thread)
 @Fork(1)
 public class EvalBenchmark {
-    private static final BigArrays BIG_ARRAYS = BigArrays.NON_RECYCLING_INSTANCE;  // TODO real big arrays?
     private static final BlockFactory blockFactory = BlockFactory.getInstance(
         new NoopCircuitBreaker("noop"),
         BigArrays.NON_RECYCLING_INSTANCE
@@ -82,7 +84,9 @@ public class EvalBenchmark {
         }
     }
 
-    @Param({ "abs", "add", "date_trunc", "equal_to_const", "long_equal_to_long", "long_equal_to_int", "mv_min", "mv_min_ascending" })
+    @Param(
+        { "abs", "add", "date_trunc", "equal_to_const", "long_equal_to_long", "long_equal_to_int", "mv_min", "mv_min_ascending", "rlike" }
+    )
     public String operation;
 
     private static Operator operator(String operation) {
@@ -134,6 +138,11 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
+            case "rlike" -> {
+                FieldAttribute keywordField = keywordField();
+                RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
+                yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext);
+            }
             default -> throw new UnsupportedOperationException();
         };
     }
@@ -146,6 +155,10 @@ private static FieldAttribute intField() {
         return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataTypes.INTEGER, Map.of(), true));
     }
 
+    private static FieldAttribute keywordField() {
+        return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataTypes.KEYWORD, Map.of(), true));
+    }
+
     private static Layout layout(FieldAttribute... fields) {
         Layout.Builder layout = new Layout.Builder();
         layout.append(Arrays.asList(fields));
@@ -205,6 +218,15 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
+            case "rlike" -> {
+                BooleanVector v = actual.<BooleanBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    boolean expected = i % 2 == 1;
+                    if (v.getBoolean(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + v.getBoolean(i) + "]");
+                    }
+                }
+            }
             default -> throw new UnsupportedOperationException();
         }
     }
@@ -250,6 +272,14 @@ private static Page page(String operation) {
                 }
                 yield new Page(builder.build());
             }
+            case "rlike" -> {
+                var builder = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
+                BytesRef[] values = new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") };
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    builder.appendBytesRef(values[i % 2]);
+                }
+                yield new Page(builder.build().asBlock());
+            }
             default -> throw new UnsupportedOperationException();
         };
     }

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.gradle.internal;
 
+import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -40,6 +41,7 @@
 
 import javax.inject.Inject;
 
+import static de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME;
 import static org.objectweb.asm.Opcodes.V_PREVIEW;
 
 public class MrjarPlugin implements Plugin<Project> {
@@ -121,6 +123,15 @@ private void addMrjarSourceset(
 
             compileTask.doLast(t -> { stripPreviewFromFiles(compileTask.getDestinationDirectory().getAsFile().get().toPath()); });
         });
+
+        // Since we configure MRJAR sourcesets to allow preview apis, class signatures for those
+        // apis are not known by forbidden apis, so we must ignore all missing classes. We could, in theory,
+        // run forbidden apis in a separate jvm matching the sourceset jvm, but it's not worth
+        // the complexity (according to forbidden apis author!)
+        String forbiddenApisTaskName = sourceSet.getTaskName(FORBIDDEN_APIS_TASK_NAME, null);
+        project.getTasks().withType(CheckForbiddenApisTask.class).named(forbiddenApisTaskName).configure(forbiddenApisTask -> {
+            forbiddenApisTask.setIgnoreMissingClasses(true);
+        });
     }
 
     private static void stripPreviewFromFiles(Path compileDir) {

diff --git a/...c/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java b/...c/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java
@@ -11,7 +11,6 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import org.apache.commons.compress.utils.Lists;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainDownload;
 import org.gradle.jvm.toolchain.JavaToolchainRequest;
@@ -21,25 +20,25 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
-import java.util.Comparator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.StreamSupport;
 
 import static org.gradle.jvm.toolchain.JavaToolchainDownload.fromUri;
 
 public abstract class AdoptiumJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
     // package protected for better testing
-    final Map<AdoptiumVersionRequest, Optional<AdoptiumVersionInfo>> CACHED_SEMVERS = new ConcurrentHashMap<>();
+    final Map<AdoptiumVersionRequest, Optional<String>> CACHED_RELEASES = new ConcurrentHashMap<>();
 
     @Override
     public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
         if (requestIsSupported(request) == false) {
             return Optional.empty();
         }
         AdoptiumVersionRequest versionRequestKey = toVersionRequest(request);
-        Optional<AdoptiumVersionInfo> versionInfo = CACHED_SEMVERS.computeIfAbsent(
+        Optional<String> versionInfo = CACHED_RELEASES.computeIfAbsent(
             versionRequestKey,
             (r) -> resolveAvailableVersion(versionRequestKey)
         );
@@ -54,12 +53,12 @@ private AdoptiumVersionRequest toVersionRequest(JavaToolchainRequest request) {
         return new AdoptiumVersionRequest(platform, arch, javaLanguageVersion);
     }
 
-    private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
+    private Optional<String> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
         ObjectMapper mapper = new ObjectMapper();
         try {
             int languageVersion = requestKey.languageVersion.asInt();
             URL source = new URL(
-                "https://api.adoptium.net/v3/info/release_versions?architecture="
+                "https://api.adoptium.net/v3/info/release_names?architecture="
                     + requestKey.arch
                     + "&image_type=jdk&os="
                     + requestKey.platform
@@ -71,14 +70,8 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
                     + ")"
             );
             JsonNode jsonNode = mapper.readTree(source);
-            JsonNode versionsNode = jsonNode.get("versions");
-            return Optional.of(
-                Lists.newArrayList(versionsNode.iterator())
-                    .stream()
-                    .map(this::toVersionInfo)
-                    .max(Comparator.comparing(AdoptiumVersionInfo::semver))
-                    .get()
-            );
+            JsonNode versionsNode = jsonNode.get("releases");
+            return StreamSupport.stream(versionsNode.spliterator(), false).map(JsonNode::textValue).findFirst();
         } catch (FileNotFoundException e) {
             // request combo not supported (e.g. aarch64 + windows
             return Optional.empty();
@@ -87,21 +80,10 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
         }
     }
 
-    private AdoptiumVersionInfo toVersionInfo(JsonNode node) {
-        return new AdoptiumVersionInfo(
-            node.get("build").asInt(),
-            node.get("major").asInt(),
-            node.get("minor").asInt(),
-            node.get("openjdk_version").asText(),
-            node.get("security").asInt(),
-            node.get("semver").asText()
-        );
-    }
-
-    private URI resolveDownloadURI(AdoptiumVersionRequest request, AdoptiumVersionInfo versionInfo) {
+    private URI resolveDownloadURI(AdoptiumVersionRequest request, String version) {
         return URI.create(
-            "https://api.adoptium.net/v3/binary/version/jdk-"
-                + versionInfo.semver
+            "https://api.adoptium.net/v3/binary/version/"
+                + version
                 + "/"
                 + request.platform
                 + "/"
@@ -118,7 +100,5 @@ private boolean requestIsSupported(JavaToolchainRequest request) {
         return anyVendorOr(request.getJavaToolchainSpec().getVendor().get(), JvmVendorSpec.ADOPTIUM);
     }
 
-    record AdoptiumVersionInfo(int build, int major, int minor, String openjdkVersion, int security, String semver) {}
-
     record AdoptiumVersionRequest(String platform, String arch, JavaLanguageVersion languageVersion) {}
 }
diff --git a/...main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java b/...main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java
@@ -39,11 +39,7 @@ record JdkBuild(JavaLanguageVersion languageVersion, String version, String buil
     );
 
     // package private so it can be replaced by tests
-    List<JdkBuild> builds = List.of(
-        getBundledJdkBuild(),
-        // 22 release candidate
-        new JdkBuild(JavaLanguageVersion.of(22), "22", "36", "830ec9fcccef480bb3e73fb7ecafe059")
-    );
+    List<JdkBuild> builds = List.of(getBundledJdkBuild());
 
     private JdkBuild getBundledJdkBuild() {
         String bundledJdkVersion = VersionProperties.getBundledJdkVersion();

diff --git a/...roovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy b/...roovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy
@@ -11,7 +11,6 @@ package org.elasticsearch.gradle.internal.toolchain
 import org.gradle.api.services.BuildServiceParameters
 import org.gradle.jvm.toolchain.JavaLanguageVersion
 import org.gradle.jvm.toolchain.JavaToolchainResolver
-import org.gradle.platform.OperatingSystem
 
 import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toArchString
 import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toOsString
@@ -38,12 +37,7 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
                     toOsString(it[2], it[1]),
                     toArchString(it[3]),
                     languageVersion);
-            resolver.CACHED_SEMVERS.put(request, Optional.of(new AdoptiumJdkToolchainResolver.AdoptiumVersionInfo(languageVersion.asInt(),
-                    1,
-                    1,
-                    "" + languageVersion.asInt() + ".1.1.1+37",
-                    0, "" + languageVersion.asInt() + ".1.1.1+37.1"
-            )))
+            resolver.CACHED_RELEASES.put(request, Optional.of('jdk-' + languageVersion.asInt() + '.1.1.1+37.1'))
 
         }
         return resolver

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -2,7 +2,7 @@ elasticsearch     = 8.14.0
 lucene            = 9.10.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac
+bundled_jdk = 22+36@830ec9fcccef480bb3e73fb7ecafe059
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

diff --git a/docs/changelog/106429.yaml b/docs/changelog/106429.yaml
@@ -0,0 +1,5 @@
+pr: 106429
+summary: "ESQL: Regex improvements"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/106435.yaml b/docs/changelog/106435.yaml
@@ -0,0 +1,6 @@
+pr: 106435
+summary: "ENRICH support for TEXT fields"
+area: ES|QL
+type: enhancement
+issues:
+ - 105384
diff --git a/docs/reference/esql/functions/abs.asciidoc b/docs/reference/esql/functions/abs.asciidoc
@@ -16,8 +16,6 @@ Numeric expression. If `null`, the function returns `null`.
 
 Returns the absolute value.
 
-*Supported types*
-
 include::types/abs.asciidoc[]
 
 *Examples*
@@ -38,4 +36,4 @@ include::{esql-specs}/math.csv-spec[tag=docsAbsEmployees]
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 include::{esql-specs}/math.csv-spec[tag=docsAbsEmployees-result]
-|===
+|===
diff --git a/docs/reference/esql/functions/acos.asciidoc b/docs/reference/esql/functions/acos.asciidoc
@@ -17,8 +17,6 @@ Numeric expression. If `null`, the function returns `null`.
 Returns the {wikipedia}/Inverse_trigonometric_functions[arccosine] of `n` as an
 angle, expressed in radians.
 
-*Supported types*
-
 include::types/acos.asciidoc[]
 
 *Example*

diff --git a/docs/reference/esql/functions/asin.asciidoc b/docs/reference/esql/functions/asin.asciidoc
@@ -17,8 +17,6 @@ Numeric expression. If `null`, the function returns `null`.
 Returns the {wikipedia}/Inverse_trigonometric_functions[arcsine] of the input
 numeric expression as an angle, expressed in radians.
 
-*Supported types*
-
 include::types/asin.asciidoc[]
 
 *Example*

diff --git a/docs/reference/esql/functions/atan.asciidoc b/docs/reference/esql/functions/atan.asciidoc
@@ -17,8 +17,6 @@ Numeric expression. If `null`, the function returns `null`.
 Returns the {wikipedia}/Inverse_trigonometric_functions[arctangent] of the input
 numeric expression as an angle, expressed in radians.
 
-*Supported types*
-
 include::types/atan.asciidoc[]
 
 *Example*
@@ -30,4 +28,4 @@ include::{esql-specs}/floats.csv-spec[tag=atan]
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 include::{esql-specs}/floats.csv-spec[tag=atan-result]
-|===
+|===
diff --git a/docs/reference/esql/functions/atan2.asciidoc b/docs/reference/esql/functions/atan2.asciidoc
@@ -20,8 +20,6 @@ Numeric expression. If `null`, the function returns `null`.
 The {wikipedia}/Atan2[angle] between the positive x-axis and the ray from the
 origin to the point (x , y) in the Cartesian plane, expressed in radians.
 
-*Supported types*
-
 include::types/atan2.asciidoc[]
 
 *Example*