From aad16b7d6b74c33733bb2ac913e0e00ace30ebf8 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 17 Aug 2023 17:29:06 +0200 Subject: [PATCH 01/12] Simple ESQL pow() docs fixes after re-reviewing (#98601) --- docs/reference/esql/functions/pow.asciidoc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/reference/esql/functions/pow.asciidoc b/docs/reference/esql/functions/pow.asciidoc index c80b64912e976..e5fb6bb2abb0e 100644 --- a/docs/reference/esql/functions/pow.asciidoc +++ b/docs/reference/esql/functions/pow.asciidoc @@ -18,8 +18,8 @@ The type of the returned value is determined by the types of the base and expone The following rules are applied to determine the result type: * If either of the base or exponent are of a floating point type, the result will be a double -* Otherwise, if either the base of the exponent are 64-bit (long or unsigned long), the result will be a long -* Otherwise, the result will be a 32-bit integer (all other numeric types, including int, short and byte) +* Otherwise, if either the base or the exponent are 64-bit (long or unsigned long), the result will be a long +* Otherwise, the result will be a 32-bit integer (this covers all other numeric types, including int, short and byte) For example, using simple integers as arguments will lead to an integer result: @@ -32,13 +32,14 @@ include::{esql-specs}/math.csv-spec[tag=powII] include::{esql-specs}/math.csv-spec[tag=powII-result] |=== -Note: The actual power function is performed using double precision values for all cases. -This means that for very large non-floating point values the operation can lead to very slightly different answers. +NOTE: The actual power function is performed using double precision values for all cases. +This means that for very large non-floating point values there is a small chance that the +operation can lead to slightly different answers than expected. However, a more likely outcome of very large non-floating point values is numerical overflow. ==== Arithmetic errors -Arithmetic errors and numeric overflow do not result in an error, instead the result will be `null` +Arithmetic errors and numeric overflow do not result in an error. Instead, the result will be `null` and a warning for the `ArithmeticException` added. For example: @@ -55,7 +56,7 @@ include::{esql-specs}/math.csv-spec[tag=powULOverrun-warning] include::{esql-specs}/math.csv-spec[tag=powULOverrun-result] |=== -If it is desired to protect against numerical overruns, use `to_double` on any one of the arguments: +If it is desired to protect against numerical overruns, use `to_double` on either of the arguments: [source.merge.styled,esql] ---- From a380e8c369095047d33e21623024112a4e435c5d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 17 Aug 2023 11:30:12 -0400 Subject: [PATCH 02/12] ESQL: LTRIM, RTRIM and fix unicode whitespace (#98590) Here we add support for the following two ESQL functions: * LTRIM: remove leading spaces from a string * RTRIM: remove trailing spaces from a string We also fix an issue with the handling of unicode white spaces. We make use of unicode code points to identify unicode whitespace characters instead of relying on ASCII codes. Moreover, iterating bytes in a Unicode string needs to consider that some Unicode characters are encoded using multiple bytes. --- .../src/main/resources/changelog-schema.json | 1 + docs/changelog/98590.yaml | 5 + docs/reference/esql/esql-functions.asciidoc | 4 + docs/reference/esql/functions/ltrim.asciidoc | 12 +++ docs/reference/esql/functions/rtrim.asciidoc | 12 +++ .../src/main/resources/show.csv-spec | 2 + .../src/main/resources/string.csv-spec | 90 ++++++++++++++--- .../scalar/string/LTrimEvaluator.java | 67 +++++++++++++ .../scalar/string/RTrimEvaluator.java | 67 +++++++++++++ .../function/EsqlFunctionRegistry.java | 4 + .../function/scalar/string/LTrim.java | 80 +++++++++++++++ .../function/scalar/string/RTrim.java | 82 ++++++++++++++++ .../function/scalar/string/Trim.java | 25 +++-- .../xpack/esql/io/stream/PlanNamedTypes.java | 6 ++ .../scalar/string/AbstractTrimTests.java | 98 +++++++++++++++++++ .../function/scalar/string/LTrimTests.java | 33 +++++++ .../function/scalar/string/RTrimTests.java | 33 +++++++ .../function/scalar/string/TrimTests.java | 59 +---------- 18 files changed, 605 insertions(+), 75 deletions(-) create mode 100644 docs/changelog/98590.yaml create mode 100644 docs/reference/esql/functions/ltrim.asciidoc create mode 100644 docs/reference/esql/functions/rtrim.asciidoc create mode 100644 x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimTests.java diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json index a784a598df363..644c543222274 100644 --- a/build-tools-internal/src/main/resources/changelog-schema.json +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -37,6 +37,7 @@ "Distributed", "Downsampling", "EQL", + "ES|QL", "Engine", "FIPS", "Features", diff --git a/docs/changelog/98590.yaml b/docs/changelog/98590.yaml new file mode 100644 index 0000000000000..f3ef3cdd56a12 --- /dev/null +++ b/docs/changelog/98590.yaml @@ -0,0 +1,5 @@ +pr: 98590 +summary: "ESQL: LTRIM, RTRIM and fix unicode whitespace" +area: ES|QL +type: feature +issues: [] diff --git a/docs/reference/esql/esql-functions.asciidoc b/docs/reference/esql/esql-functions.asciidoc index 2a4b64331390b..27581bd6ae57d 100644 --- a/docs/reference/esql/esql-functions.asciidoc +++ b/docs/reference/esql/esql-functions.asciidoc @@ -30,6 +30,8 @@ these functions: * <> * <> * <> +* <> +* <> * <> * <> * <> @@ -85,6 +87,8 @@ include::functions/is_infinite.asciidoc[] include::functions/is_nan.asciidoc[] include::functions/length.asciidoc[] include::functions/log10.asciidoc[] +include::functions/ltrim.asciidoc[] +include::functions/rtrim.asciidoc[] include::functions/mv_avg.asciidoc[] include::functions/mv_concat.asciidoc[] include::functions/mv_count.asciidoc[] diff --git a/docs/reference/esql/functions/ltrim.asciidoc b/docs/reference/esql/functions/ltrim.asciidoc new file mode 100644 index 0000000000000..7a809d85f14f5 --- /dev/null +++ b/docs/reference/esql/functions/ltrim.asciidoc @@ -0,0 +1,12 @@ +[[esql-ltrim]] +=== `LTRIM` +Removes leading whitespaces from strings. + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=ltrim] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=ltrim-result] +|=== diff --git a/docs/reference/esql/functions/rtrim.asciidoc b/docs/reference/esql/functions/rtrim.asciidoc new file mode 100644 index 0000000000000..916a6596b2e2e --- /dev/null +++ b/docs/reference/esql/functions/rtrim.asciidoc @@ -0,0 +1,12 @@ +[[esql-rtrim]] +=== `RTRIM` +Removes trailing whitespaces from strings. + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=rtrim] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=rtrim-result] +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index 99991d39cf8d3..1f7bdd3fc6957 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -34,6 +34,7 @@ is_infinite |is_infinite(arg1) is_nan |is_nan(arg1) length |length(arg1) log10 |log10(arg1) +ltrim |ltrim(arg1) max |max(arg1) median |median(arg1) median_absolute_deviation|median_absolute_deviation(arg1) @@ -51,6 +52,7 @@ percentile |percentile(arg1, arg2) pi |pi() pow |pow(arg1, arg2) round |round(arg1, arg2) +rtrim |rtrim(arg1) sin |sin(arg1) sinh |sinh(arg1) split |split(arg1, arg2) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 2dbceb3b12335..51bd57d9dc1da 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -165,25 +165,91 @@ emp_no:integer | last_name:keyword | x:keyword | z:keyword 10010 | Piveteau | P | a ; +ltrim +from employees | sort emp_no | limit 10 | eval name = concat(" ", first_name, " ") | eval name = ltrim(name) | eval name = concat("'", name, "'") | keep emp_no, name; + +emp_no:integer | name:keyword +10001 | 'Georgi ' +10002 | 'Bezalel ' +10003 | 'Parto ' +10004 | 'Chirstian ' +10005 | 'Kyoichi ' +10006 | 'Anneke ' +10007 | 'Tzvetan ' +10008 | 'Saniya ' +10009 | 'Sumant ' +10010 | 'Duangkaew ' +; + +ltrimRow +// tag::ltrim[] +ROW message = " some text ", color = " red " +| EVAL message = LTRIM(message) +| EVAL color = LTRIM(color) +| EVAL message = CONCAT("'", message, "'") +| EVAL color = CONCAT("'", color, "'") +// end::ltrim[] +; + +// tag::ltrim-result[] +message:keyword | color:keyword +'some text ' | 'red ' +// end::ltrim-result[] +; + +rtrim +from employees | sort emp_no | limit 10 | eval name = concat(" ", first_name, " ") | eval name = rtrim(name) | eval name = concat("'", name, "'") | keep emp_no, name; + +emp_no:integer | name:keyword +10001 | ' Georgi' +10002 | ' Bezalel' +10003 | ' Parto' +10004 | ' Chirstian' +10005 | ' Kyoichi' +10006 | ' Anneke' +10007 | ' Tzvetan' +10008 | ' Saniya' +10009 | ' Sumant' +10010 | ' Duangkaew' +; + +rtrimRow +// tag::rtrim[] +ROW message = " some text ", color = " red " +| EVAL message = RTRIM(message) +| EVAL color = RTRIM(color) +| EVAL message = CONCAT("'", message, "'") +| EVAL color = CONCAT("'", color, "'") +// end::rtrim[] +; + +// tag::rtrim-result[] +message:keyword | color:keyword +' some text' | ' red' +// end::rtrim-result[] +; + trim -from employees | sort emp_no | limit 10 | eval name = concat(" ", first_name) | eval name = trim(first_name) | keep emp_no, name; +from employees | sort emp_no | limit 10 | eval name = concat(" ", first_name, " ") | eval name = trim(name) | eval name = concat("'", name, "'") | keep emp_no, name; emp_no:integer | name:keyword -10001 | Georgi -10002 | Bezalel -10003 | Parto -10004 | Chirstian -10005 | Kyoichi -10006 | Anneke -10007 | Tzvetan -10008 | Saniya -10009 | Sumant -10010 | Duangkaew +10001 | 'Georgi' +10002 | 'Bezalel' +10003 | 'Parto' +10004 | 'Chirstian' +10005 | 'Kyoichi' +10006 | 'Anneke' +10007 | 'Tzvetan' +10008 | 'Saniya' +10009 | 'Sumant' +10010 | 'Duangkaew' ; trimRow // tag::trim[] -ROW message = " some text ", color = " red "| EVAL message = TRIM(message)| EVAL color = TRIM(color) +ROW message = " some text ", color = " red " +| EVAL message = TRIM(message) +| EVAL color = TRIM(color) // end::trim[] ; diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java new file mode 100644 index 0000000000000..19bb7f4cb4e6a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java @@ -0,0 +1,67 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.EvalOperator; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link LTrim}. + * This class is generated. Do not edit it. + */ +public final class LTrimEvaluator implements EvalOperator.ExpressionEvaluator { + private final EvalOperator.ExpressionEvaluator val; + + public LTrimEvaluator(EvalOperator.ExpressionEvaluator val) { + this.val = val; + } + + @Override + public Block eval(Page page) { + Block valUncastBlock = val.eval(page); + if (valUncastBlock.areAllValuesNull()) { + return Block.constantNullBlock(page.getPositionCount()); + } + BytesRefBlock valBlock = (BytesRefBlock) valUncastBlock; + BytesRefVector valVector = valBlock.asVector(); + if (valVector == null) { + return eval(page.getPositionCount(), valBlock); + } + return eval(page.getPositionCount(), valVector).asBlock(); + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) { + BytesRefBlock.Builder result = BytesRefBlock.newBlockBuilder(positionCount); + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valBlock.isNull(p) || valBlock.getValueCount(p) != 1) { + result.appendNull(); + continue position; + } + result.appendBytesRef(LTrim.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch))); + } + return result.build(); + } + + public BytesRefVector eval(int positionCount, BytesRefVector valVector) { + BytesRefVector.Builder result = BytesRefVector.newVectorBuilder(positionCount); + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(LTrim.process(valVector.getBytesRef(p, valScratch))); + } + return result.build(); + } + + @Override + public String toString() { + return "LTrimEvaluator[" + "val=" + val + "]"; + } +} diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java new file mode 100644 index 0000000000000..946542427c53b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java @@ -0,0 +1,67 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.EvalOperator; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link RTrim}. + * This class is generated. Do not edit it. + */ +public final class RTrimEvaluator implements EvalOperator.ExpressionEvaluator { + private final EvalOperator.ExpressionEvaluator val; + + public RTrimEvaluator(EvalOperator.ExpressionEvaluator val) { + this.val = val; + } + + @Override + public Block eval(Page page) { + Block valUncastBlock = val.eval(page); + if (valUncastBlock.areAllValuesNull()) { + return Block.constantNullBlock(page.getPositionCount()); + } + BytesRefBlock valBlock = (BytesRefBlock) valUncastBlock; + BytesRefVector valVector = valBlock.asVector(); + if (valVector == null) { + return eval(page.getPositionCount(), valBlock); + } + return eval(page.getPositionCount(), valVector).asBlock(); + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) { + BytesRefBlock.Builder result = BytesRefBlock.newBlockBuilder(positionCount); + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valBlock.isNull(p) || valBlock.getValueCount(p) != 1) { + result.appendNull(); + continue position; + } + result.appendBytesRef(RTrim.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch))); + } + return result.build(); + } + + public BytesRefVector eval(int positionCount, BytesRefVector valVector) { + BytesRefVector.Builder result = BytesRefVector.newVectorBuilder(positionCount); + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(RTrim.process(valVector.getBytesRef(p, valScratch))); + } + return result.build(); + } + + @Override + public String toString() { + return "RTrimEvaluator[" + "val=" + val + "]"; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 9a16435684648..1d1a1f4591056 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -66,7 +66,9 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; @@ -129,6 +131,8 @@ private FunctionDefinition[][] functions() { def(Length.class, Length::new, "length"), def(Substring.class, Substring::new, "substring"), def(Concat.class, Concat::new, "concat"), + def(LTrim.class, LTrim::new, "ltrim"), + def(RTrim.class, RTrim::new, "rtrim"), def(Trim.class, Trim::new, "trim"), def(StartsWith.class, StartsWith::new, "starts_with") }, // date diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java new file mode 100644 index 0000000000000..0ae643cc96a8c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.planner.Mappable; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; + +/** + * Removes leading whitespaces from a string. + */ +public class LTrim extends UnaryScalarFunction implements Mappable { + public LTrim(Source source, Expression field) { + super(source, field); + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new Expression.TypeResolution("Unresolved children"); + } + + return isString(field, sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public Object fold() { + return Mappable.super.fold(); + } + + @Override + public Supplier toEvaluator( + Function> toEvaluator + ) { + Supplier field = toEvaluator.apply(field()); + return () -> new LTrimEvaluator(field.get()); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new LTrim(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, LTrim::new, field()); + } + + @Evaluator + static BytesRef process(final BytesRef val) { + int offset = val.offset; + UnicodeUtil.UTF8CodePoint codePoint = new UnicodeUtil.UTF8CodePoint(); + while (offset < val.offset + val.length) { + codePoint = UnicodeUtil.codePointAt(val.bytes, offset, codePoint); + if (Character.isWhitespace(codePoint.codePoint) == false) { + break; + } + offset += codePoint.numBytes; + } + return new BytesRef(val.bytes, offset, val.length + val.offset - offset); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java new file mode 100644 index 0000000000000..df7c9b18900cc --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.planner.Mappable; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; + +/** + * Removes trailing whitespaces from a string. + */ +public class RTrim extends UnaryScalarFunction implements Mappable { + public RTrim(Source source, Expression field) { + super(source, field); + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new Expression.TypeResolution("Unresolved children"); + } + + return isString(field, sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public Object fold() { + return Mappable.super.fold(); + } + + @Override + public Supplier toEvaluator( + Function> toEvaluator + ) { + Supplier field = toEvaluator.apply(field()); + return () -> new RTrimEvaluator(field.get()); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new RTrim(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, RTrim::new, field()); + } + + @Evaluator + static BytesRef process(final BytesRef val) { + int offset = val.offset; + int end = offset; + UnicodeUtil.UTF8CodePoint codePoint = new UnicodeUtil.UTF8CodePoint(); + while (offset < val.offset + val.length) { + codePoint = UnicodeUtil.codePointAt(val.bytes, offset, codePoint); + if (Character.isWhitespace(codePoint.codePoint) == false) { + end = offset + codePoint.numBytes; + } + offset += codePoint.numBytes; + } + + return new BytesRef(val.bytes, val.offset, end - val.offset); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java index c9a454eb16105..818c78ccb0ae6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; @@ -67,13 +68,25 @@ protected NodeInfo info() { @Evaluator static BytesRef process(BytesRef val) { int offset = val.offset; - int length = val.length; - while ((offset < length) && ((val.bytes[offset] & 0xff) <= 0x20)) { - offset++; + UnicodeUtil.UTF8CodePoint codePoint = new UnicodeUtil.UTF8CodePoint(); + while (offset < val.offset + val.length) { + codePoint = UnicodeUtil.codePointAt(val.bytes, offset, codePoint); + if (Character.isWhitespace(codePoint.codePoint) == false) { + break; + } + offset += codePoint.numBytes; } - while ((offset < length) && ((val.bytes[length - 1] & 0xff) <= 0x20)) { - length--; + + int end = offset; + int i = offset; + while (i < val.offset + val.length) { + codePoint = UnicodeUtil.codePointAt(val.bytes, i, codePoint); + if (Character.isWhitespace(codePoint.codePoint) == false) { + end = i + codePoint.numBytes; + } + i += codePoint.numBytes; } - return new BytesRef(val.bytes, offset, length - (offset - val.offset)); + + return new BytesRef(val.bytes, offset, end - offset); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 1ea0fa690880b..f94f3677fe3cc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -77,7 +77,9 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; @@ -299,6 +301,8 @@ public static List namedTypeEntries() { of(ESQL_UNARY_SCLR_CLS, IsNaN.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), of(ESQL_UNARY_SCLR_CLS, Length.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), of(ESQL_UNARY_SCLR_CLS, Log10.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), + of(ESQL_UNARY_SCLR_CLS, LTrim.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), + of(ESQL_UNARY_SCLR_CLS, RTrim.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), of(ESQL_UNARY_SCLR_CLS, Sin.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), of(ESQL_UNARY_SCLR_CLS, Sinh.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), of(ESQL_UNARY_SCLR_CLS, Sqrt.class, PlanNamedTypes::writeESQLUnaryScalar, PlanNamedTypes::readESQLUnaryScalar), @@ -1025,6 +1029,8 @@ static void writeBinaryLogic(PlanStreamOutput out, BinaryLogic binaryLogic) thro entry(name(IsNaN.class), IsNaN::new), entry(name(Length.class), Length::new), entry(name(Log10.class), Log10::new), + entry(name(LTrim.class), LTrim::new), + entry(name(RTrim.class), RTrim::new), entry(name(Sin.class), Sin::new), entry(name(Sinh.class), Sinh::new), entry(name(Sqrt.class), Sqrt::new), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java new file mode 100644 index 0000000000000..c5f7448edd6f6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.ql.type.DataType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractTrimTests extends AbstractScalarFunctionTestCase { + static Iterable parameters(String name, boolean trimLeading, boolean trimTrailing) { + List suppliers = new ArrayList<>(); + for (DataType type : strings()) { + suppliers.add(new TestCaseSupplier("no whitespace/" + type, () -> { + String text = randomAlphaOfLength(8); + return testCase(name, type, text, text); + })); + + for (Map.Entry whitespaces : List.of( + Map.entry("spaces", new char[] { ' ' }), + Map.entry("tabs", new char[] { '\t' }), + Map.entry("newlines", new char[] { '\n' }), + Map.entry("line tabulation", new char[] { '\u000B' }), + Map.entry("form feed", new char[] { '\f' }), + Map.entry("carriage return", new char[] { '\r' }), + Map.entry("file separator", new char[] { '\u001C' }), + Map.entry("group separator", new char[] { '\u001D' }), + Map.entry("information separator two", new char[] { '\u001E' }), + Map.entry("information separator one", new char[] { '\u001F' }), + Map.entry("whitespace", new char[] { ' ', '\t', '\n', '\u000B', '\f', '\r', '\u001C', '\u001D', '\u001E', '\u001F' }) + )) { + suppliers.add(new TestCaseSupplier(type + "/leading " + whitespaces.getKey(), () -> { + String text = randomAlphaOfLength(8); + String withWhitespace = randomWhiteSpace(whitespaces.getValue()) + text; + return testCase(name, type, withWhitespace, trimLeading ? text : withWhitespace); + })); + suppliers.add(new TestCaseSupplier(type + "/trailing " + whitespaces.getKey(), () -> { + String text = randomAlphaOfLength(8); + String withWhitespace = text + randomWhiteSpace(whitespaces.getValue()); + return testCase(name, type, withWhitespace, trimTrailing ? text : withWhitespace); + })); + suppliers.add(new TestCaseSupplier(type + "/leading and trailing " + whitespaces.getKey(), () -> { + String text = randomAlphaOfLength(8); + String leadingWhitespace = randomWhiteSpace(whitespaces.getValue()); + String trailingWhitespace = randomWhiteSpace(whitespaces.getValue()); + return testCase( + name, + type, + leadingWhitespace + text + trailingWhitespace, + (trimLeading ? "" : leadingWhitespace) + text + (trimTrailing ? "" : trailingWhitespace) + ); + })); + suppliers.add(new TestCaseSupplier(type + "/all " + whitespaces.getKey(), () -> { + String text = randomWhiteSpace(whitespaces.getValue()); + return testCase(name, type, text, ""); + })); + } + } + return parameterSuppliersFromTypedData(suppliers); + } + + private static TestCase testCase(String name, DataType type, String data, String expected) { + return new TestCase( + List.of(new TypedData(new BytesRef(data), type, "str")), + name + "[val=Attribute[channel=0]]", + type, + equalTo(new BytesRef(expected)) + ); + } + + @Override + protected final List argSpec() { + return List.of(required(strings())); + } + + @Override + protected final DataType expectedType(List argTypes) { + return argTypes.get(0); + } + + private static String randomWhiteSpace(char[] whitespaces) { + char[] randomWhitespace = new char[randomIntBetween(1, 8)]; + for (int i = 0; i < randomWhitespace.length; i++) { + randomWhitespace[i] = whitespaces[randomIntBetween(0, whitespaces.length - 1)]; + } + return new String(randomWhitespace); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimTests.java new file mode 100644 index 0000000000000..74ae9a09244d5 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; +import java.util.function.Supplier; + +public class LTrimTests extends AbstractTrimTests { + public LTrimTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + return parameters("LTrimEvaluator", true, false); + } + + @Override + protected Expression build(Source source, List args) { + return new LTrim(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimTests.java new file mode 100644 index 0000000000000..a6017f5162e7e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; +import java.util.function.Supplier; + +public class RTrimTests extends AbstractTrimTests { + public RTrimTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + return parameters("RTrimEvaluator", false, true); + } + + @Override + protected Expression build(Source source, List args) { + return new RTrim(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimTests.java index 1a8bf4da893bd..422d4da9dd121 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimTests.java @@ -10,79 +10,24 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import java.util.Arrays; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - -public class TrimTests extends AbstractScalarFunctionTestCase { +public class TrimTests extends AbstractTrimTests { public TrimTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Trim basic test", () -> { - BytesRef sampleData = addRandomLeadingOrTrailingWhitespaces(randomUnicodeOfLength(8)); - DataType dataType = randomFrom(strings()); - return new TestCase( - List.of(new TypedData(sampleData, dataType, "str")), - "TrimEvaluator[val=Attribute[channel=0]]", - dataType, - equalTo(new BytesRef(sampleData.utf8ToString().trim())) - ); - }))); + return parameters("TrimEvaluator", true, true); } @Override protected Expression build(Source source, List args) { return new Trim(source, args.get(0)); } - - @Override - protected List argSpec() { - return List.of(required(strings())); - } - - @Override - protected DataType expectedType(List argTypes) { - return argTypes.get(0); - } - - public void testTrim() { - for (int i = 0; i < 64; i++) { - String expected = randomUnicodeOfLength(8).trim(); - BytesRef result = Trim.process(addRandomLeadingOrTrailingWhitespaces(expected)); - assertThat(result.utf8ToString(), equalTo(expected)); - } - } - - static BytesRef addRandomLeadingOrTrailingWhitespaces(String expected) { - StringBuilder builder = new StringBuilder(); - if (randomBoolean()) { - builder.append(randomWhiteSpace()); - builder.append(expected); - if (randomBoolean()) { - builder.append(randomWhiteSpace()); - } - } else { - builder.append(expected); - builder.append(randomWhiteSpace()); - } - return new BytesRef(builder.toString()); - } - - private static char[] randomWhiteSpace() { - char[] randomWhitespace = new char[randomIntBetween(1, 8)]; - Arrays.fill(randomWhitespace, (char) randomIntBetween(0, 0x20)); - return randomWhitespace; - } - } From 79d2879564398ef878f80414f129dad5ae3bc7bb Mon Sep 17 00:00:00 2001 From: Roberto Seldner Date: Thu, 17 Aug 2023 09:50:52 -0700 Subject: [PATCH 03/12] Add deprecated note for `balanced` allocator (#98610) Co-authored-by: James Rodewig --- docs/reference/modules/cluster/shards_allocation.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc index 42282e00e81d4..1f7b83294651f 100644 --- a/docs/reference/modules/cluster/shards_allocation.asciidoc +++ b/docs/reference/modules/cluster/shards_allocation.asciidoc @@ -115,6 +115,7 @@ runs a background task which computes the desired balance of shards in the cluster. Once this background task completes, {es} moves shards to their desired locations. +deprecated:[8.8,The `balanced` allocator type is deprecated and no longer recommended] May also be set to `balanced` to select the legacy _balanced allocator_. This allocator was the default allocator in versions of {es} before 8.6.0. It runs in the foreground, preventing the master from doing other work in parallel. It From 51e4618c7a3b60146a9cac81ddf3c7d6a96e8e9f Mon Sep 17 00:00:00 2001 From: Volodymyr Krasnikov Date: Thu, 17 Aug 2023 09:51:53 -0700 Subject: [PATCH 04/12] Bump versions after 8.9.1 release --- .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- server/src/main/java/org/elasticsearch/Version.java | 1 + server/src/main/java/org/elasticsearch/index/IndexVersion.java | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 65b7f54bca8db..5d337dd10a9bc 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -94,5 +94,6 @@ BWC_VERSION: - "8.8.2" - "8.9.0" - "8.9.1" + - "8.9.2" - "8.10.0" - "8.11.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index f25f0ae5619d1..6ad91910ab7b0 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - "7.17.13" - - "8.9.1" + - "8.9.2" - "8.10.0" - "8.11.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index e8d89ec4d12f0..2dda35b69c6ec 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -147,6 +147,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_8_2 = new Version(8_08_02_99, IndexVersion.V_8_8_2); public static final Version V_8_9_0 = new Version(8_09_00_99, IndexVersion.V_8_9_0); public static final Version V_8_9_1 = new Version(8_09_01_99, IndexVersion.V_8_9_1); + public static final Version V_8_9_2 = new Version(8_09_02_99, IndexVersion.V_8_9_2); public static final Version V_8_10_0 = new Version(8_10_00_99, IndexVersion.V_8_10_0); public static final Version V_8_11_0 = new Version(8_11_00_99, IndexVersion.V_8_11_0); public static final Version CURRENT = V_8_11_0; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersion.java b/server/src/main/java/org/elasticsearch/index/IndexVersion.java index 90feb9288eb1c..a70bea87850ad 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersion.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersion.java @@ -178,6 +178,7 @@ private static IndexVersion registerIndexVersion(int id, Version luceneVersion, public static final IndexVersion V_8_8_2 = registerIndexVersion(8_08_02_99, Version.LUCENE_9_6_0, "9db9d888-6be8-4a58-825c-f423fd8c6b00"); public static final IndexVersion V_8_9_0 = registerIndexVersion(8_09_00_99, Version.LUCENE_9_7_0, "32f6dbab-cc24-4f5b-87b5-015a848480d9"); public static final IndexVersion V_8_9_1 = registerIndexVersion(8_09_01_99, Version.LUCENE_9_7_0, "955a80ac-f70c-40a5-9399-1d8a1e5d342d"); + public static final IndexVersion V_8_9_2 = registerIndexVersion(8_09_02_99, Version.LUCENE_9_7_0, "14c7d64c-9e25-4265-b4fa-e0c5aca67f14"); public static final IndexVersion V_8_10_0 = registerIndexVersion(8_10_00_99, Version.LUCENE_9_7_0, "2e107286-12ad-4c51-9a6f-f8943663b6e7"); public static final IndexVersion V_8_11_0 = registerIndexVersion(8_11_00_99, Version.LUCENE_9_7_0, "f08382c0-06ab-41f4-a56a-cf5397275627"); /* From 07afba669f50449030c0c8e5c78265235cb503f3 Mon Sep 17 00:00:00 2001 From: Volodymyr Krasnikov Date: Thu, 17 Aug 2023 09:54:55 -0700 Subject: [PATCH 05/12] Prune changelogs after 8.9.1 release --- docs/changelog/88590.yaml | 6 ------ docs/changelog/92060.yaml | 6 ------ docs/changelog/92348.yaml | 5 ----- docs/changelog/94089.yaml | 5 ----- docs/changelog/94510.yaml | 5 ----- docs/changelog/94534.yaml | 5 ----- docs/changelog/94859.yaml | 5 ----- docs/changelog/94954.yaml | 25 ------------------------- docs/changelog/95107.yaml | 5 ----- docs/changelog/95112.yaml | 5 ----- docs/changelog/95204.yaml | 5 ----- docs/changelog/95286.yaml | 5 ----- docs/changelog/95329.yaml | 6 ------ docs/changelog/95430.yaml | 5 ----- docs/changelog/95459.yaml | 5 ----- docs/changelog/95481.yaml | 6 ------ docs/changelog/95512.yaml | 5 ----- docs/changelog/95548.yaml | 5 ----- docs/changelog/95575.yaml | 5 ----- docs/changelog/95625.yaml | 6 ------ docs/changelog/95666.yaml | 5 ----- docs/changelog/95672.yaml | 5 ----- docs/changelog/95674.yaml | 5 ----- docs/changelog/95694.yaml | 5 ----- docs/changelog/95705.yaml | 6 ------ docs/changelog/95731.yaml | 5 ----- docs/changelog/95734.yaml | 5 ----- docs/changelog/95740.yaml | 6 ------ docs/changelog/95782.yaml | 5 ----- docs/changelog/95811.yaml | 5 ----- docs/changelog/95828.yaml | 5 ----- docs/changelog/95874.yaml | 6 ------ docs/changelog/95894.yaml | 6 ------ docs/changelog/95910.yaml | 5 ----- docs/changelog/95921.yaml | 5 ----- docs/changelog/95971.yaml | 7 ------- docs/changelog/95979.yaml | 5 ----- docs/changelog/95987.yaml | 8 -------- docs/changelog/95996.yaml | 5 ----- docs/changelog/96012.yaml | 5 ----- docs/changelog/96051.yaml | 5 ----- docs/changelog/96056.yaml | 5 ----- docs/changelog/96083.yaml | 6 ------ docs/changelog/96092.yaml | 5 ----- docs/changelog/96161.yaml | 31 ------------------------------- docs/changelog/96171.yaml | 6 ------ docs/changelog/96177.yaml | 5 ----- docs/changelog/96180.yaml | 6 ------ docs/changelog/96197.yaml | 5 ----- docs/changelog/96198.yaml | 6 ------ docs/changelog/96224.yaml | 14 -------------- docs/changelog/96262.yaml | 5 ----- docs/changelog/96268.yaml | 5 ----- docs/changelog/96272.yaml | 5 ----- docs/changelog/96279.yaml | 5 ----- docs/changelog/96281.yaml | 5 ----- docs/changelog/96321.yaml | 6 ------ docs/changelog/96328.yaml | 6 ------ docs/changelog/96340.yaml | 5 ----- docs/changelog/96363.yaml | 5 ----- docs/changelog/96394.yaml | 6 ------ docs/changelog/96399.yaml | 5 ----- docs/changelog/96406.yaml | 5 ----- docs/changelog/96407.yaml | 6 ------ docs/changelog/96421.yaml | 6 ------ docs/changelog/96433.yaml | 5 ----- docs/changelog/96453.yaml | 5 ----- docs/changelog/96458.yaml | 5 ----- docs/changelog/96479.yaml | 6 ------ docs/changelog/96490.yaml | 6 ------ docs/changelog/96516.yaml | 5 ----- docs/changelog/96550.yaml | 5 ----- docs/changelog/96588.yaml | 5 ----- docs/changelog/96605.yaml | 6 ------ docs/changelog/96613.yaml | 6 ------ docs/changelog/96617.yaml | 5 ----- docs/changelog/96624.yaml | 5 ----- docs/changelog/96662.yaml | 6 ------ docs/changelog/96678.yaml | 6 ------ docs/changelog/96712.yaml | 5 ----- docs/changelog/96716.yaml | 5 ----- docs/changelog/96741.yaml | 5 ----- docs/changelog/96744.yaml | 5 ----- docs/changelog/96777.yaml | 7 ------- docs/changelog/96790.yaml | 5 ----- docs/changelog/96794.yaml | 5 ----- docs/changelog/96824.yaml | 5 ----- docs/changelog/96834.yaml | 5 ----- docs/changelog/96863.yaml | 5 ----- docs/changelog/96878.yaml | 5 ----- docs/changelog/96885.yaml | 5 ----- docs/changelog/96904.yaml | 18 ------------------ docs/changelog/96924.yaml | 6 ------ docs/changelog/96941.yaml | 6 ------ docs/changelog/96943.yaml | 5 ----- docs/changelog/96953.yaml | 6 ------ docs/changelog/96956.yaml | 6 ------ docs/changelog/96958.yaml | 5 ----- docs/changelog/96970.yaml | 5 ----- docs/changelog/97038.yaml | 6 ------ docs/changelog/97062.yaml | 6 ------ docs/changelog/97099.yaml | 5 ----- docs/changelog/97111.yaml | 5 ----- docs/changelog/97119.yaml | 6 ------ docs/changelog/97248.yaml | 6 ------ docs/changelog/97251.yaml | 6 ------ docs/changelog/97274.yaml | 5 ----- docs/changelog/97278.yaml | 6 ------ docs/changelog/97290.yaml | 6 ------ docs/changelog/97304.yaml | 5 ----- docs/changelog/97353.yaml | 6 ------ docs/changelog/97354.yaml | 6 ------ docs/changelog/97355.yaml | 6 ------ docs/changelog/97401.yaml | 5 ----- docs/changelog/97460.yaml | 6 ------ docs/changelog/97509.yaml | 6 ------ docs/changelog/97587.yaml | 5 ----- docs/changelog/97619.yaml | 6 ------ docs/changelog/97679.yaml | 5 ----- docs/changelog/97683.yaml | 5 ----- docs/changelog/97741.yaml | 5 ----- docs/changelog/97775.yaml | 5 ----- docs/changelog/97840.yaml | 6 ------ docs/changelog/97936.yaml | 6 ------ docs/changelog/97979.yaml | 5 ----- docs/changelog/97987.yaml | 6 ------ docs/changelog/98091.yaml | 6 ------ docs/changelog/98113.yaml | 5 ----- docs/changelog/98167.yaml | 6 ------ docs/changelog/98176.yaml | 5 ----- docs/changelog/98296.yaml | 5 ----- 131 files changed, 776 deletions(-) delete mode 100644 docs/changelog/88590.yaml delete mode 100644 docs/changelog/92060.yaml delete mode 100644 docs/changelog/92348.yaml delete mode 100644 docs/changelog/94089.yaml delete mode 100644 docs/changelog/94510.yaml delete mode 100644 docs/changelog/94534.yaml delete mode 100644 docs/changelog/94859.yaml delete mode 100644 docs/changelog/94954.yaml delete mode 100644 docs/changelog/95107.yaml delete mode 100644 docs/changelog/95112.yaml delete mode 100644 docs/changelog/95204.yaml delete mode 100644 docs/changelog/95286.yaml delete mode 100644 docs/changelog/95329.yaml delete mode 100644 docs/changelog/95430.yaml delete mode 100644 docs/changelog/95459.yaml delete mode 100644 docs/changelog/95481.yaml delete mode 100644 docs/changelog/95512.yaml delete mode 100644 docs/changelog/95548.yaml delete mode 100644 docs/changelog/95575.yaml delete mode 100644 docs/changelog/95625.yaml delete mode 100644 docs/changelog/95666.yaml delete mode 100644 docs/changelog/95672.yaml delete mode 100644 docs/changelog/95674.yaml delete mode 100644 docs/changelog/95694.yaml delete mode 100644 docs/changelog/95705.yaml delete mode 100644 docs/changelog/95731.yaml delete mode 100644 docs/changelog/95734.yaml delete mode 100644 docs/changelog/95740.yaml delete mode 100644 docs/changelog/95782.yaml delete mode 100644 docs/changelog/95811.yaml delete mode 100644 docs/changelog/95828.yaml delete mode 100644 docs/changelog/95874.yaml delete mode 100644 docs/changelog/95894.yaml delete mode 100644 docs/changelog/95910.yaml delete mode 100644 docs/changelog/95921.yaml delete mode 100644 docs/changelog/95971.yaml delete mode 100644 docs/changelog/95979.yaml delete mode 100644 docs/changelog/95987.yaml delete mode 100644 docs/changelog/95996.yaml delete mode 100644 docs/changelog/96012.yaml delete mode 100644 docs/changelog/96051.yaml delete mode 100644 docs/changelog/96056.yaml delete mode 100644 docs/changelog/96083.yaml delete mode 100644 docs/changelog/96092.yaml delete mode 100644 docs/changelog/96161.yaml delete mode 100644 docs/changelog/96171.yaml delete mode 100644 docs/changelog/96177.yaml delete mode 100644 docs/changelog/96180.yaml delete mode 100644 docs/changelog/96197.yaml delete mode 100644 docs/changelog/96198.yaml delete mode 100644 docs/changelog/96224.yaml delete mode 100644 docs/changelog/96262.yaml delete mode 100644 docs/changelog/96268.yaml delete mode 100644 docs/changelog/96272.yaml delete mode 100644 docs/changelog/96279.yaml delete mode 100644 docs/changelog/96281.yaml delete mode 100644 docs/changelog/96321.yaml delete mode 100644 docs/changelog/96328.yaml delete mode 100644 docs/changelog/96340.yaml delete mode 100644 docs/changelog/96363.yaml delete mode 100644 docs/changelog/96394.yaml delete mode 100644 docs/changelog/96399.yaml delete mode 100644 docs/changelog/96406.yaml delete mode 100644 docs/changelog/96407.yaml delete mode 100644 docs/changelog/96421.yaml delete mode 100644 docs/changelog/96433.yaml delete mode 100644 docs/changelog/96453.yaml delete mode 100644 docs/changelog/96458.yaml delete mode 100644 docs/changelog/96479.yaml delete mode 100644 docs/changelog/96490.yaml delete mode 100644 docs/changelog/96516.yaml delete mode 100644 docs/changelog/96550.yaml delete mode 100644 docs/changelog/96588.yaml delete mode 100644 docs/changelog/96605.yaml delete mode 100644 docs/changelog/96613.yaml delete mode 100644 docs/changelog/96617.yaml delete mode 100644 docs/changelog/96624.yaml delete mode 100644 docs/changelog/96662.yaml delete mode 100644 docs/changelog/96678.yaml delete mode 100644 docs/changelog/96712.yaml delete mode 100644 docs/changelog/96716.yaml delete mode 100644 docs/changelog/96741.yaml delete mode 100644 docs/changelog/96744.yaml delete mode 100644 docs/changelog/96777.yaml delete mode 100644 docs/changelog/96790.yaml delete mode 100644 docs/changelog/96794.yaml delete mode 100644 docs/changelog/96824.yaml delete mode 100644 docs/changelog/96834.yaml delete mode 100644 docs/changelog/96863.yaml delete mode 100644 docs/changelog/96878.yaml delete mode 100644 docs/changelog/96885.yaml delete mode 100644 docs/changelog/96904.yaml delete mode 100644 docs/changelog/96924.yaml delete mode 100644 docs/changelog/96941.yaml delete mode 100644 docs/changelog/96943.yaml delete mode 100644 docs/changelog/96953.yaml delete mode 100644 docs/changelog/96956.yaml delete mode 100644 docs/changelog/96958.yaml delete mode 100644 docs/changelog/96970.yaml delete mode 100644 docs/changelog/97038.yaml delete mode 100644 docs/changelog/97062.yaml delete mode 100644 docs/changelog/97099.yaml delete mode 100644 docs/changelog/97111.yaml delete mode 100644 docs/changelog/97119.yaml delete mode 100644 docs/changelog/97248.yaml delete mode 100644 docs/changelog/97251.yaml delete mode 100644 docs/changelog/97274.yaml delete mode 100644 docs/changelog/97278.yaml delete mode 100644 docs/changelog/97290.yaml delete mode 100644 docs/changelog/97304.yaml delete mode 100644 docs/changelog/97353.yaml delete mode 100644 docs/changelog/97354.yaml delete mode 100644 docs/changelog/97355.yaml delete mode 100644 docs/changelog/97401.yaml delete mode 100644 docs/changelog/97460.yaml delete mode 100644 docs/changelog/97509.yaml delete mode 100644 docs/changelog/97587.yaml delete mode 100644 docs/changelog/97619.yaml delete mode 100644 docs/changelog/97679.yaml delete mode 100644 docs/changelog/97683.yaml delete mode 100644 docs/changelog/97741.yaml delete mode 100644 docs/changelog/97775.yaml delete mode 100644 docs/changelog/97840.yaml delete mode 100644 docs/changelog/97936.yaml delete mode 100644 docs/changelog/97979.yaml delete mode 100644 docs/changelog/97987.yaml delete mode 100644 docs/changelog/98091.yaml delete mode 100644 docs/changelog/98113.yaml delete mode 100644 docs/changelog/98167.yaml delete mode 100644 docs/changelog/98176.yaml delete mode 100644 docs/changelog/98296.yaml diff --git a/docs/changelog/88590.yaml b/docs/changelog/88590.yaml deleted file mode 100644 index 4ec4434318067..0000000000000 --- a/docs/changelog/88590.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 88590 -summary: Add shard explain info to `ReactiveReason` about unassigned shards -area: Autoscaling -type: enhancement -issues: - - 85243 diff --git a/docs/changelog/92060.yaml b/docs/changelog/92060.yaml deleted file mode 100644 index 02f1144234578..0000000000000 --- a/docs/changelog/92060.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 92060 -summary: Add support for dynamic pruning to cardinality aggregations on low-cardinality - keyword fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/92348.yaml b/docs/changelog/92348.yaml deleted file mode 100644 index 6199f8e0ada8d..0000000000000 --- a/docs/changelog/92348.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 92348 -summary: Sequence - add support for missing events -area: EQL -type: enhancement -issues: [] diff --git a/docs/changelog/94089.yaml b/docs/changelog/94089.yaml deleted file mode 100644 index 7acc4324ff91f..0000000000000 --- a/docs/changelog/94089.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 94089 -summary: Add support for `xlm_roberta` tokenized models -area: Machine Learning -type: feature -issues: [] diff --git a/docs/changelog/94510.yaml b/docs/changelog/94510.yaml deleted file mode 100644 index 91e80c9aadbb6..0000000000000 --- a/docs/changelog/94510.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 94510 -summary: Reduce WaitForNoFollowersStep requests indices shard stats -area: ILM+SLM -type: enhancement -issues: [] diff --git a/docs/changelog/94534.yaml b/docs/changelog/94534.yaml deleted file mode 100644 index 0a15b46c7099b..0000000000000 --- a/docs/changelog/94534.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 94534 -summary: Add search shards endpoint -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/94859.yaml b/docs/changelog/94859.yaml deleted file mode 100644 index 18f199fd345db..0000000000000 --- a/docs/changelog/94859.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 94859 -summary: Geometry simplifier -area: Geo -type: feature -issues: [] diff --git a/docs/changelog/94954.yaml b/docs/changelog/94954.yaml deleted file mode 100644 index aaac45153bef8..0000000000000 --- a/docs/changelog/94954.yaml +++ /dev/null @@ -1,25 +0,0 @@ -pr: 94954 -summary: Asset tracking - geo_line in time-series aggregations -area: Geo -type: enhancement -issues: [ ] -highlight: - title: Asset tracking - geo_line in time-series aggregations - body: |- - The <> builds tracks from `geo_points`. - It has previously needed to use large arrays in memory for collecting points into multiple buckets - and sorting those buckets. - - With the advances made in TSDB features and the `time_series` aggregation in particular, - it is now possible to rely on data aggregating in both TSID and timestamp order, - enabling the removal of all sorting, as well as the use of only a single bucket's - worth of memory, a dramatic improvement in memory footprint. In addition, we can use the streaming line - simplifier algorithm introduced in https://github.com/elastic/elasticsearch/pull/94859 to replace the previous - behaviour of truncating very large tracks with the far more preferable approach of simplifying those tracks. - - [role="screenshot"] - image:images/spatial/kodiak_geo_line_simplified.png[North short of Kodiak Island simplified to 100 points] - - In this diagram, the grey line is the original geometry, the blue line is the truncated geometry as would be - produced by the original `geo_line` aggregation, and the magenta line is the new simplified geometry. - notable: false diff --git a/docs/changelog/95107.yaml b/docs/changelog/95107.yaml deleted file mode 100644 index 9b757e0d68042..0000000000000 --- a/docs/changelog/95107.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95107 -summary: Cache modification time of translog writer file -area: Engine -type: enhancement -issues: [] diff --git a/docs/changelog/95112.yaml b/docs/changelog/95112.yaml deleted file mode 100644 index 030cc1be308da..0000000000000 --- a/docs/changelog/95112.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95112 -summary: Header validator with Security -area: Authentication -type: enhancement -issues: [] diff --git a/docs/changelog/95204.yaml b/docs/changelog/95204.yaml deleted file mode 100644 index 3e114f90af9b5..0000000000000 --- a/docs/changelog/95204.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95204 -summary: Add auto force merge functionality to DLM -area: DLM -type: enhancement -issues: [] diff --git a/docs/changelog/95286.yaml b/docs/changelog/95286.yaml deleted file mode 100644 index 1677d5ecb9335..0000000000000 --- a/docs/changelog/95286.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95286 -summary: Bump `TransportVersion` to the first non-release version number. Transport protocol is now versioned independently of release version. -area: Infra/Transport API -type: upgrade -issues: [] diff --git a/docs/changelog/95329.yaml b/docs/changelog/95329.yaml deleted file mode 100644 index 891a2494651ed..0000000000000 --- a/docs/changelog/95329.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 95329 -summary: Change default of `ignore_malformed` to `true` in `logs-*-*` data streams -area: Data streams -type: enhancement -issues: - - 95224 diff --git a/docs/changelog/95430.yaml b/docs/changelog/95430.yaml deleted file mode 100644 index 2d3242583dcef..0000000000000 --- a/docs/changelog/95430.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95430 -summary: SIGTERM node shutdown type -area: Infra/Node Lifecycle -type: enhancement -issues: [] diff --git a/docs/changelog/95459.yaml b/docs/changelog/95459.yaml deleted file mode 100644 index eb3b7c13f0d62..0000000000000 --- a/docs/changelog/95459.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95459 -summary: Add slf4j-nop in order to prevent startup warnings -area: Infra/Logging -type: bug -issues: [] diff --git a/docs/changelog/95481.yaml b/docs/changelog/95481.yaml deleted file mode 100644 index 2789566326fec..0000000000000 --- a/docs/changelog/95481.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 95481 -summary: "Add support for `logs@custom` component template for `logs-*-* data streams" -area: Data streams -type: enhancement -issues: - - 95469 diff --git a/docs/changelog/95512.yaml b/docs/changelog/95512.yaml deleted file mode 100644 index 4df5cd5de4588..0000000000000 --- a/docs/changelog/95512.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95512 -summary: Adding `manage_data_stream_lifecycle` index privilege and expanding `view_index_metadata` for access to data stream lifecycle APIs -area: DLM -type: enhancement -issues: [] diff --git a/docs/changelog/95548.yaml b/docs/changelog/95548.yaml deleted file mode 100644 index 64529ba38aa5b..0000000000000 --- a/docs/changelog/95548.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95548 -summary: Create `.synonyms` system index -area: Analysis -type: enhancement -issues: [] diff --git a/docs/changelog/95575.yaml b/docs/changelog/95575.yaml deleted file mode 100644 index d7003f24ab10d..0000000000000 --- a/docs/changelog/95575.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95575 -summary: Upgrade Netty to 4.1.92 -area: Network -type: upgrade -issues: [] diff --git a/docs/changelog/95625.yaml b/docs/changelog/95625.yaml deleted file mode 100644 index 264116a8478a7..0000000000000 --- a/docs/changelog/95625.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 95625 -summary: "[Fleet] Add `.fleet-secrets` system index" -area: Infra/Plugins -type: feature -issues: - - 95143 diff --git a/docs/changelog/95666.yaml b/docs/changelog/95666.yaml deleted file mode 100644 index 5f8d5c62fc05e..0000000000000 --- a/docs/changelog/95666.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95666 -summary: Bootstrap profiling indices at startup -area: Indices APIs -type: enhancement -issues: [] diff --git a/docs/changelog/95672.yaml b/docs/changelog/95672.yaml deleted file mode 100644 index a43aaedbe9401..0000000000000 --- a/docs/changelog/95672.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95672 -summary: Improve reporting status of the transform that is about to finish -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/95674.yaml b/docs/changelog/95674.yaml deleted file mode 100644 index f91cded49acf7..0000000000000 --- a/docs/changelog/95674.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95674 -summary: Add template parameters to Search Applications -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/95694.yaml b/docs/changelog/95694.yaml deleted file mode 100644 index 5e48f41a84591..0000000000000 --- a/docs/changelog/95694.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95694 -summary: Add "_storage" internal user -area: Security -type: enhancement -issues: [] diff --git a/docs/changelog/95705.yaml b/docs/changelog/95705.yaml deleted file mode 100644 index dcd33dbcc61d8..0000000000000 --- a/docs/changelog/95705.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 95705 -summary: Avoid stack overflow while parsing mapping -area: Mapping -type: bug -issues: - - 52098 diff --git a/docs/changelog/95731.yaml b/docs/changelog/95731.yaml deleted file mode 100644 index fca60ac274faa..0000000000000 --- a/docs/changelog/95731.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95731 -summary: Implement `StartRecoveryRequest#getDescription` -area: Recovery -type: enhancement -issues: [] diff --git a/docs/changelog/95734.yaml b/docs/changelog/95734.yaml deleted file mode 100644 index b6fb33f0356b9..0000000000000 --- a/docs/changelog/95734.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95734 -summary: Take into account `expectedShardSize` when initializing shard in simulation -area: Allocation -type: enhancement -issues: [] diff --git a/docs/changelog/95740.yaml b/docs/changelog/95740.yaml deleted file mode 100644 index a477efcbd83cb..0000000000000 --- a/docs/changelog/95740.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 95740 -summary: Include search idle info to shard stats -area: "Search" -type: enhancement -issues: - - 95727 diff --git a/docs/changelog/95782.yaml b/docs/changelog/95782.yaml deleted file mode 100644 index 5e1e0d86636bc..0000000000000 --- a/docs/changelog/95782.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95782 -summary: Adding ability to auto-install ingest pipelines and refer to them from index templates -area: Data streams -type: feature -issues: [] diff --git a/docs/changelog/95811.yaml b/docs/changelog/95811.yaml deleted file mode 100644 index 10c6fe7e4ad6a..0000000000000 --- a/docs/changelog/95811.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95811 -summary: Support null_value for rank_feature field type -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/95828.yaml b/docs/changelog/95828.yaml deleted file mode 100644 index ebe327432128b..0000000000000 --- a/docs/changelog/95828.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95828 -summary: Support value retrieval in `top_hits` -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/95874.yaml b/docs/changelog/95874.yaml deleted file mode 100644 index b55b0410c6df3..0000000000000 --- a/docs/changelog/95874.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 95874 -summary: Enable validation for `versionSettings` -area: "Infra/Settings" -type: bug -issues: - - 95873 diff --git a/docs/changelog/95894.yaml b/docs/changelog/95894.yaml deleted file mode 100644 index 2ac1125a564a7..0000000000000 --- a/docs/changelog/95894.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 95894 -summary: Integrate CCS with new `search_shards` API -area: Search -type: enhancement -issues: - - 93730 diff --git a/docs/changelog/95910.yaml b/docs/changelog/95910.yaml deleted file mode 100644 index e3126aa292d9a..0000000000000 --- a/docs/changelog/95910.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95910 -summary: Don't generate stacktrace in `EarlyTerminationException` and `TimeExceededException` -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/95921.yaml b/docs/changelog/95921.yaml deleted file mode 100644 index c0d32120f1686..0000000000000 --- a/docs/changelog/95921.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95921 -summary: Attempt to fix delay allocation -area: Allocation -type: bug -issues: [] diff --git a/docs/changelog/95971.yaml b/docs/changelog/95971.yaml deleted file mode 100644 index 584f5ce00348c..0000000000000 --- a/docs/changelog/95971.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 95971 -summary: "Set `@timestamp` for documents in logs data streams if missing and add support for custom pipeline" -area: Data streams -type: enhancement -issues: - - 95537 - - 95551 diff --git a/docs/changelog/95979.yaml b/docs/changelog/95979.yaml deleted file mode 100644 index 87e247b514ee1..0000000000000 --- a/docs/changelog/95979.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95979 -summary: Allow for the data stream lifecycle and the retention to be explicitly nullified -area: DLM -type: enhancement -issues: [] diff --git a/docs/changelog/95987.yaml b/docs/changelog/95987.yaml deleted file mode 100644 index 91ccbf82abce5..0000000000000 --- a/docs/changelog/95987.yaml +++ /dev/null @@ -1,8 +0,0 @@ -pr: 95987 -summary: Fix reused/recovered bytes for files that are only partially recovered from - cache -area: Snapshot/Restore -type: bug -issues: - - 95970 - - 95994 diff --git a/docs/changelog/95996.yaml b/docs/changelog/95996.yaml deleted file mode 100644 index 725e546e79c0d..0000000000000 --- a/docs/changelog/95996.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 95996 -summary: Fixing `DateProcessor` when the format is `epoch_millis` -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/96012.yaml b/docs/changelog/96012.yaml deleted file mode 100644 index 818867f3fe8e6..0000000000000 --- a/docs/changelog/96012.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96012 -summary: Support CCS minimize round trips in async search -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/96051.yaml b/docs/changelog/96051.yaml deleted file mode 100644 index 30283774f7a74..0000000000000 --- a/docs/changelog/96051.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96051 -summary: Update data streams implicit timestamp `ignore_malformed` settings -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/96056.yaml b/docs/changelog/96056.yaml deleted file mode 100644 index a32a6dd675bb9..0000000000000 --- a/docs/changelog/96056.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96056 -summary: Add mappings for enrich fields -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/96083.yaml b/docs/changelog/96083.yaml deleted file mode 100644 index dcbf843767399..0000000000000 --- a/docs/changelog/96083.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96083 -summary: Automatically parse log events in logs data streams, if their `message` field contains JSON content -area: Data streams -type: enhancement -issues: - - 95522 diff --git a/docs/changelog/96092.yaml b/docs/changelog/96092.yaml deleted file mode 100644 index f632ebe3966b3..0000000000000 --- a/docs/changelog/96092.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96092 -summary: Enhance ILM Health Indicator -area: ILM+SLM -type: feature -issues: [] diff --git a/docs/changelog/96161.yaml b/docs/changelog/96161.yaml deleted file mode 100644 index 66368c403a94d..0000000000000 --- a/docs/changelog/96161.yaml +++ /dev/null @@ -1,31 +0,0 @@ -pr: 96161 -summary: Skip shards when querying constant keyword fields -area: "Search" -type: enhancement -issues: - - 95541 -highlight: - title: Better indexing and search performance under concurrent indexing and search - body: "When a query like a match phrase query or a terms query targets a constant keyword field we can skip\ - query execution on shards where the query is rewritten to match no documents. We take advantage of index mappings\ - including constant keyword fields and rewrite queries in such a way that, if a constant keyword field does not\ - match the value defined in the index mapping, we rewrite the query to match no document. This will result in the\ - shard level request to return immediately, before the query is executed on the data node and, as a result, skipping\ - the shard completely. Here we leverage the ability to skip shards whenever possible to avoid unnecessary shard\ - refreshes and improve query latency (by not doing any search-related I/O). Avoiding such unnecessary shard refreshes\ - improves query latency since the search thread does not need to wait anymore for unnecessary shard refreshes. Shards\ - not matching the query criteria will remain in a search-idle state and indexing throughput will not be negatively\ - affected by a refresh. Before introducing this change a query hitting multiple shards, including those with no\ - documents matching the search criteria (think about using index patterns or data streams with many backing indices),\ - would potentially result in a \"shard refresh storm\" increasing query latency as a result of the search thread\ - waiting on all shard refreshes to complete before being able to initiate and carry out the search operation.\ - After introducing this change the search thread will just need to wait for refreshes to be completed on shards\ - including relevant data. Note that execution of the shard pre-filter and the corresponding \"can match\" phase where\ - rewriting happens, depends on the overall number of shards involved and on whether there is at least one of them\ - returning a non-empty result (see 'pre_filter_shard_size' setting to understand how to control this behaviour).\ - Elasticsearch does the rewrite operation on the data node in the so called \"can match\" phase, taking advantage of\ - the fact that, at that moment, we can access index mappings and extract information about constant keyword fields\ - and their values. This means we still\"fan-out\" search queries from the coordinator node to involved data nodes.\ - Rewriting queries based on index mappings is not possible on the coordinator node because the coordinator node is\ - missing index mapping information." - notable: true diff --git a/docs/changelog/96171.yaml b/docs/changelog/96171.yaml deleted file mode 100644 index 8a170ad819a11..0000000000000 --- a/docs/changelog/96171.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96171 -summary: "Adding ECS dynamic mappings component and applying it to logs data streams by default" -area: Data streams -type: enhancement -issues: - - 95538 diff --git a/docs/changelog/96177.yaml b/docs/changelog/96177.yaml deleted file mode 100644 index d832a99019fda..0000000000000 --- a/docs/changelog/96177.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96177 -summary: Adding `data_lifecycle` to the _xpack/usage API -area: DLM -type: enhancement -issues: [] diff --git a/docs/changelog/96180.yaml b/docs/changelog/96180.yaml deleted file mode 100644 index c87a0dd75266e..0000000000000 --- a/docs/changelog/96180.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96180 -summary: Prevent instantiation of `top_metrics` when sub-aggregations are present -area: Search -type: bug -issues: - - 95663 diff --git a/docs/changelog/96197.yaml b/docs/changelog/96197.yaml deleted file mode 100644 index b93928e587206..0000000000000 --- a/docs/changelog/96197.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96197 -summary: "[Search Applications] Support arrays in stored mustache templates" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/96198.yaml b/docs/changelog/96198.yaml deleted file mode 100644 index 17b7e2e75cd8e..0000000000000 --- a/docs/changelog/96198.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96198 -summary: New HTTP info endpoint -area: Stats -type: feature -issues: - - 95391 diff --git a/docs/changelog/96224.yaml b/docs/changelog/96224.yaml deleted file mode 100644 index 71500e689765c..0000000000000 --- a/docs/changelog/96224.yaml +++ /dev/null @@ -1,14 +0,0 @@ -pr: 96224 -summary: Add multiple queries for ranking to the search endpoint -area: Ranking -type: enhancement -issues: [] -highlight: - title: Add multiple queries for ranking to the search endpoint - body: "The search endpoint adds a new top-level element called `sub_searches`. \ - This top-level element is a list of searches used for ranking where each \ - \"sub search\" is executed independently. The `sub_searches` element is \ - used instead of `query` to allow a search using ranking to execute \ - multiple queries. The `sub_searches` and `query` elements cannot be used \ - together." - notable: true diff --git a/docs/changelog/96262.yaml b/docs/changelog/96262.yaml deleted file mode 100644 index 190c76ef607d8..0000000000000 --- a/docs/changelog/96262.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96262 -summary: Fix `retry_on_conflict` parameter in update API to not retry indefinitely -area: CRUD -type: bug -issues: [] diff --git a/docs/changelog/96268.yaml b/docs/changelog/96268.yaml deleted file mode 100644 index 4504a356adec9..0000000000000 --- a/docs/changelog/96268.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96268 -summary: "[Profiling] Introduce ILM for K/V indices" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/96272.yaml b/docs/changelog/96272.yaml deleted file mode 100644 index 739983e8f9d46..0000000000000 --- a/docs/changelog/96272.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96272 -summary: "[Profiling] Add status API" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/96279.yaml b/docs/changelog/96279.yaml deleted file mode 100644 index 39c14d64e34a4..0000000000000 --- a/docs/changelog/96279.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96279 -summary: Improve cancellability in `TransportTasksAction` -area: Task Management -type: bug -issues: [] diff --git a/docs/changelog/96281.yaml b/docs/changelog/96281.yaml deleted file mode 100644 index 9913f72c0364e..0000000000000 --- a/docs/changelog/96281.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96281 -summary: Fix autoexpand during node replace -area: Allocation -type: bug -issues: [] diff --git a/docs/changelog/96321.yaml b/docs/changelog/96321.yaml deleted file mode 100644 index f15abf282bbb6..0000000000000 --- a/docs/changelog/96321.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96321 -summary: Trigger refresh when shard becomes search active -area: Engine -type: enhancement -issues: - - 95544 diff --git a/docs/changelog/96328.yaml b/docs/changelog/96328.yaml deleted file mode 100644 index 91f83959a76d6..0000000000000 --- a/docs/changelog/96328.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96328 -summary: Add `ingest` information to the cluster info endpoint -area: Stats -type: enhancement -issues: - - 95392 diff --git a/docs/changelog/96340.yaml b/docs/changelog/96340.yaml deleted file mode 100644 index c0fc9133d7370..0000000000000 --- a/docs/changelog/96340.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96340 -summary: Chunk profiling stacktrace response -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/96363.yaml b/docs/changelog/96363.yaml deleted file mode 100644 index 567ffae298f74..0000000000000 --- a/docs/changelog/96363.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96363 -summary: Gracefully shutdown elasticsearch -area: Infra/Node Lifecycle -type: feature -issues: [] diff --git a/docs/changelog/96394.yaml b/docs/changelog/96394.yaml deleted file mode 100644 index f23c81e97b7f9..0000000000000 --- a/docs/changelog/96394.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96394 -summary: Allow unsigned long field to use decay functions -area: Mapping -type: enhancement -issues: - - 89603 diff --git a/docs/changelog/96399.yaml b/docs/changelog/96399.yaml deleted file mode 100644 index d553795ea16a0..0000000000000 --- a/docs/changelog/96399.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96399 -summary: Reduce overhead in blob cache service get -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/96406.yaml b/docs/changelog/96406.yaml deleted file mode 100644 index 8ceddff3dfbef..0000000000000 --- a/docs/changelog/96406.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96406 -summary: Fix tchar pattern in `RestRequest` -area: Infra/REST API -type: bug -issues: [] diff --git a/docs/changelog/96407.yaml b/docs/changelog/96407.yaml deleted file mode 100644 index 2ce6e2caf71cf..0000000000000 --- a/docs/changelog/96407.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96407 -summary: Add `thread_pool` information to the cluster info endpoint -area: Stats -type: enhancement -issues: - - 95393 diff --git a/docs/changelog/96421.yaml b/docs/changelog/96421.yaml deleted file mode 100644 index a628b3b157f98..0000000000000 --- a/docs/changelog/96421.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96421 -summary: Promptly fail recovery from snapshot -area: Recovery -type: bug -issues: - - 95525 diff --git a/docs/changelog/96433.yaml b/docs/changelog/96433.yaml deleted file mode 100644 index 04e1e9451637b..0000000000000 --- a/docs/changelog/96433.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96433 -summary: Upgrade Lucene to a 9.7.0 snapshot -area: Search -type: upgrade -issues: [] diff --git a/docs/changelog/96453.yaml b/docs/changelog/96453.yaml deleted file mode 100644 index 575fa036a4088..0000000000000 --- a/docs/changelog/96453.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96453 -summary: Leverage SIMD hardware instructions in Vector Search -area: Vector Search -type: enhancement -issues: [96370] diff --git a/docs/changelog/96458.yaml b/docs/changelog/96458.yaml deleted file mode 100644 index ac6420cbc4144..0000000000000 --- a/docs/changelog/96458.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96458 -summary: Fix blob cache races/assertion errors -area: Snapshot/Restore -type: bug -issues: [] diff --git a/docs/changelog/96479.yaml b/docs/changelog/96479.yaml deleted file mode 100644 index 57d290dce0fd3..0000000000000 --- a/docs/changelog/96479.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96479 -summary: Resolving wildcard application names without prefix query -area: Authorization -type: bug -issues: - - 96465 diff --git a/docs/changelog/96490.yaml b/docs/changelog/96490.yaml deleted file mode 100644 index 169af207d8fbd..0000000000000 --- a/docs/changelog/96490.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96490 -summary: Become candidate on publication failure -area: Cluster Coordination -type: bug -issues: - - 96273 diff --git a/docs/changelog/96516.yaml b/docs/changelog/96516.yaml deleted file mode 100644 index df07b5d997497..0000000000000 --- a/docs/changelog/96516.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96516 -summary: "[Profiling] Tweak default ILM policy" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/96550.yaml b/docs/changelog/96550.yaml deleted file mode 100644 index cd7770ef1a926..0000000000000 --- a/docs/changelog/96550.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96550 -summary: "[Profiling] Allow to upgrade managed ILM policy" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/96588.yaml b/docs/changelog/96588.yaml deleted file mode 100644 index fe6a07abc424f..0000000000000 --- a/docs/changelog/96588.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96588 -summary: Support for patter_replace filter in keyword normalizer -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/96605.yaml b/docs/changelog/96605.yaml deleted file mode 100644 index f35ee0264c32c..0000000000000 --- a/docs/changelog/96605.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96605 -summary: "Feature: include unit support for time series rate aggregation" -area: TSDB -type: enhancement -issues: - - 94630 diff --git a/docs/changelog/96613.yaml b/docs/changelog/96613.yaml deleted file mode 100644 index 143c2cc15a1fd..0000000000000 --- a/docs/changelog/96613.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96613 -summary: Add `script` information to the cluster info endpoint -area: Stats -type: enhancement -issues: - - 95394 diff --git a/docs/changelog/96617.yaml b/docs/changelog/96617.yaml deleted file mode 100644 index d5ccb38c71436..0000000000000 --- a/docs/changelog/96617.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96617 -summary: Improve brute force vector search speed by using Lucene functions -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/96624.yaml b/docs/changelog/96624.yaml deleted file mode 100644 index a9e8b5e9ab6bb..0000000000000 --- a/docs/changelog/96624.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96624 -summary: Enable analytics geoip in behavioral analytics -area: "Application" -type: feature -issues: [] diff --git a/docs/changelog/96662.yaml b/docs/changelog/96662.yaml deleted file mode 100644 index e9e983e0e9343..0000000000000 --- a/docs/changelog/96662.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96662 -summary: Refactor `RestoreClusterStateListener` to use `ClusterStateObserver` -area: Snapshot/Restore -type: bug -issues: - - 96425 diff --git a/docs/changelog/96678.yaml b/docs/changelog/96678.yaml deleted file mode 100644 index a9994a218efb0..0000000000000 --- a/docs/changelog/96678.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96678 -summary: Add repo throttle metrics to node stats api response -area: Snapshot/Restore -type: feature -issues: - - 89385 diff --git a/docs/changelog/96712.yaml b/docs/changelog/96712.yaml deleted file mode 100644 index 7e303a560177d..0000000000000 --- a/docs/changelog/96712.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96712 -summary: "Adjust ECS dynamic templates to support `subobjects: false`" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/96716.yaml b/docs/changelog/96716.yaml deleted file mode 100644 index 051e64a9207ce..0000000000000 --- a/docs/changelog/96716.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96716 -summary: Feature/speed up binary vector decoding -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/96741.yaml b/docs/changelog/96741.yaml deleted file mode 100644 index d7d5cdfa93a13..0000000000000 --- a/docs/changelog/96741.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96741 -summary: Upgrade to new lucene snapshot 9.7.0-snapshot-a8602d6ef88 -area: Search -type: upgrade -issues: [] diff --git a/docs/changelog/96744.yaml b/docs/changelog/96744.yaml deleted file mode 100644 index 1f3d33f391664..0000000000000 --- a/docs/changelog/96744.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96744 -summary: Support restricting access of API keys to only certain workflows -area: Authorization -type: feature -issues: [] diff --git a/docs/changelog/96777.yaml b/docs/changelog/96777.yaml deleted file mode 100644 index 71df384088b0e..0000000000000 --- a/docs/changelog/96777.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 96777 -summary: Fixing `GeoIpDownloaderStatsAction$NodeResponse` serialization by defensively - copying inputs -area: Ingest Node -type: bug -issues: - - 96438 diff --git a/docs/changelog/96790.yaml b/docs/changelog/96790.yaml deleted file mode 100644 index 00a288823d48e..0000000000000 --- a/docs/changelog/96790.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96790 -summary: "[Profiling] Require POST to retrieve stacktraces" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/96794.yaml b/docs/changelog/96794.yaml deleted file mode 100644 index 955aa2cff5c25..0000000000000 --- a/docs/changelog/96794.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96794 -summary: Make TDigestState configurable -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/96824.yaml b/docs/changelog/96824.yaml deleted file mode 100644 index b3399f5bc4d09..0000000000000 --- a/docs/changelog/96824.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96824 -summary: Introduce a filtered collector manager -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/96834.yaml b/docs/changelog/96834.yaml deleted file mode 100644 index 73fb0ef33f83b..0000000000000 --- a/docs/changelog/96834.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96834 -summary: Introduce minimum score collector manager -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/96863.yaml b/docs/changelog/96863.yaml deleted file mode 100644 index 8bafe6d21f9bf..0000000000000 --- a/docs/changelog/96863.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96863 -summary: Add brute force approach to `GeoHashGridTiler` -area: Geo -type: enhancement -issues: [] diff --git a/docs/changelog/96878.yaml b/docs/changelog/96878.yaml deleted file mode 100644 index 7086ffd182b5b..0000000000000 --- a/docs/changelog/96878.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96878 -summary: Min score for time series -area: TSDB -type: bug -issues: [] diff --git a/docs/changelog/96885.yaml b/docs/changelog/96885.yaml deleted file mode 100644 index 0187f4ead7f2f..0000000000000 --- a/docs/changelog/96885.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96885 -summary: Add Search ALC filter index prefix to the enterprise search user -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/96904.yaml b/docs/changelog/96904.yaml deleted file mode 100644 index c4cd6aedbaa5d..0000000000000 --- a/docs/changelog/96904.yaml +++ /dev/null @@ -1,18 +0,0 @@ -pr: 96904 -summary: Switch TDigestState to use `HybridDigest` by default -area: Aggregations -type: breaking -issues: [] -breaking: - title: Switch TDigestState to use `HybridDigest` by default - area: REST API - details: "The default implementation for TDigest in percentile calculations switches\ - \ to a new internal implementation offering superior performance (2x-10x speedup),\ - \ at a very small accuracy penalty for very large sample populations." - impact: "This change leads to generating slightly different results in percentile\ - \ calculations. If the highest possible accuracy is desired, or it's crucial to\ - \ produce exactly the same results as in previous versions, one can either set\ - \ `execution_hint` to `high_accuracy` in the `tdigest` spec of a given percentile\ - \ calculation, or set `search.aggs.tdigest_execution_hint` to `high_accuracy`\ - \ in cluster settings to apply to all percentile queries." - notable: true diff --git a/docs/changelog/96924.yaml b/docs/changelog/96924.yaml deleted file mode 100644 index 0bd4abee00293..0000000000000 --- a/docs/changelog/96924.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96924 -summary: Interpret microseconds cpu stats from cgroups2 properly as nanos -area: Infra/Core -type: bug -issues: - - 96089 diff --git a/docs/changelog/96941.yaml b/docs/changelog/96941.yaml deleted file mode 100644 index dc342e7a51779..0000000000000 --- a/docs/changelog/96941.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96941 -summary: Trim field references in reroute processor -area: Ingest Node -type: bug -issues: - - 96939 diff --git a/docs/changelog/96943.yaml b/docs/changelog/96943.yaml deleted file mode 100644 index 5f8987654f4e5..0000000000000 --- a/docs/changelog/96943.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96943 -summary: Add cluster setting to `SearchExecutionContext` to configure `TDigestExecutionHint` -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/96953.yaml b/docs/changelog/96953.yaml deleted file mode 100644 index 36689374897b7..0000000000000 --- a/docs/changelog/96953.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96953 -summary: Fix time-series geo_line to include reduce phase in MergedGeoLines -area: Geo -type: bug -issues: - - 96983 diff --git a/docs/changelog/96956.yaml b/docs/changelog/96956.yaml deleted file mode 100644 index 6bd27c822e120..0000000000000 --- a/docs/changelog/96956.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 96956 -summary: Error message for misconfigured TSDB index -area: TSDB -type: bug -issues: - - 96445 diff --git a/docs/changelog/96958.yaml b/docs/changelog/96958.yaml deleted file mode 100644 index 14bb653713127..0000000000000 --- a/docs/changelog/96958.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96958 -summary: "Ingest: expose reroute inquiry/reset via Elastic-internal API bridge" -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/96970.yaml b/docs/changelog/96970.yaml deleted file mode 100644 index 6a752cc5cd224..0000000000000 --- a/docs/changelog/96970.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 96970 -summary: Ensure checking application privileges work with nested-limited roles -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/97038.yaml b/docs/changelog/97038.yaml deleted file mode 100644 index b7db20b6d1973..0000000000000 --- a/docs/changelog/97038.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97038 -summary: Limit the details field length we store for each SLM invocation -area: ILM+SLM -type: bug -issues: - - 96918 diff --git a/docs/changelog/97062.yaml b/docs/changelog/97062.yaml deleted file mode 100644 index 5320efaf48ec3..0000000000000 --- a/docs/changelog/97062.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97062 -summary: Fix Painless method lookup over unknown super interfaces -area: Infra/Scripting -type: bug -issues: - - 97022 diff --git a/docs/changelog/97099.yaml b/docs/changelog/97099.yaml deleted file mode 100644 index 1f5bc4062b8c5..0000000000000 --- a/docs/changelog/97099.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97099 -summary: Skip `SortingDigest` when merging a large digest in `HybridDigest` -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/97111.yaml b/docs/changelog/97111.yaml deleted file mode 100644 index cd5810f1cdf09..0000000000000 --- a/docs/changelog/97111.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97111 -summary: Fix cluster settings update task acknowledgment -area: Cluster Coordination -type: bug -issues: [] diff --git a/docs/changelog/97119.yaml b/docs/changelog/97119.yaml deleted file mode 100644 index 8341aa68c7329..0000000000000 --- a/docs/changelog/97119.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97119 -summary: Capture max processors in static init -area: Infra/Core -type: bug -issues: - - 97088 diff --git a/docs/changelog/97248.yaml b/docs/changelog/97248.yaml deleted file mode 100644 index c81f515c97105..0000000000000 --- a/docs/changelog/97248.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97248 -summary: Move get lifecycle API to Management thread pool and make cancellable -area: ILM+SLM -type: enhancement -issues: - - 96568 diff --git a/docs/changelog/97251.yaml b/docs/changelog/97251.yaml deleted file mode 100644 index 8cfb50befbb7f..0000000000000 --- a/docs/changelog/97251.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97251 -summary: Chunk the GET _ilm/policy response -area: ILM+SLM -type: enhancement -issues: - - 96569 diff --git a/docs/changelog/97274.yaml b/docs/changelog/97274.yaml deleted file mode 100644 index 537e013024c1e..0000000000000 --- a/docs/changelog/97274.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97274 -summary: Improve model downloader robustness -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/97278.yaml b/docs/changelog/97278.yaml deleted file mode 100644 index 5619c52711053..0000000000000 --- a/docs/changelog/97278.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97278 -summary: Fix reused/recovered bytes for files that are recovered from cache -area: Snapshot/Restore -type: bug -issues: - - 95994 diff --git a/docs/changelog/97290.yaml b/docs/changelog/97290.yaml deleted file mode 100644 index e2bbc03f331db..0000000000000 --- a/docs/changelog/97290.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97290 -summary: Handle failure in `TransportUpdateAction#handleUpdateFailureWithRetry` -area: CRUD -type: bug -issues: - - 97286 diff --git a/docs/changelog/97304.yaml b/docs/changelog/97304.yaml deleted file mode 100644 index e3d53750c5e11..0000000000000 --- a/docs/changelog/97304.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97304 -summary: Avoid `getStateForMasterService` where possible -area: Cluster Coordination -type: bug -issues: [] diff --git a/docs/changelog/97353.yaml b/docs/changelog/97353.yaml deleted file mode 100644 index c4db52a0686ef..0000000000000 --- a/docs/changelog/97353.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97353 -summary: Initialise ES logging in CLI -area: Infra/CLI -type: bug -issues: - - 97350 diff --git a/docs/changelog/97354.yaml b/docs/changelog/97354.yaml deleted file mode 100644 index 40848f87e6dbc..0000000000000 --- a/docs/changelog/97354.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97354 -summary: Fix `WordPiece` tokenization where stripping accents results in an empty - string -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/97355.yaml b/docs/changelog/97355.yaml deleted file mode 100644 index 8f899f67847a6..0000000000000 --- a/docs/changelog/97355.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97355 -summary: Fix mapping parsing logic to determine synthetic source is active -area: "Mapping" -type: bug -issues: - - 97320 diff --git a/docs/changelog/97401.yaml b/docs/changelog/97401.yaml deleted file mode 100644 index 27e712af1d730..0000000000000 --- a/docs/changelog/97401.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97401 -summary: Accept timestamp as object at root level -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/97460.yaml b/docs/changelog/97460.yaml deleted file mode 100644 index 55bcacaaded11..0000000000000 --- a/docs/changelog/97460.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97460 -summary: Set new providers before building `FetchSubPhaseProcessors` -area: "Search" -type: bug -issues: - - 96284 diff --git a/docs/changelog/97509.yaml b/docs/changelog/97509.yaml deleted file mode 100644 index dd371f65fd658..0000000000000 --- a/docs/changelog/97509.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97509 -summary: Fix bug when creating empty `geo_lines` -area: Geo -type: bug -issues: - - 97311 diff --git a/docs/changelog/97587.yaml b/docs/changelog/97587.yaml deleted file mode 100644 index 39906bc4a4412..0000000000000 --- a/docs/changelog/97587.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97587 -summary: Fix `sub_searches` serialization bug -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/97619.yaml b/docs/changelog/97619.yaml deleted file mode 100644 index 1c17e4e513691..0000000000000 --- a/docs/changelog/97619.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97619 -summary: Support for Byte and Short as vector tiles features -area: Geo -type: bug -issues: - - 97612 diff --git a/docs/changelog/97679.yaml b/docs/changelog/97679.yaml deleted file mode 100644 index d8c96fcba4acb..0000000000000 --- a/docs/changelog/97679.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97679 -summary: Add Configuration to `PatternLayout` -area: Infra/Logging -type: bug -issues: [] diff --git a/docs/changelog/97683.yaml b/docs/changelog/97683.yaml deleted file mode 100644 index 8b843eb7e9cf6..0000000000000 --- a/docs/changelog/97683.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97683 -summary: Refactor nested field handling in `FieldFetcher` -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/97741.yaml b/docs/changelog/97741.yaml deleted file mode 100644 index 926e933e7c838..0000000000000 --- a/docs/changelog/97741.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97741 -summary: Upgrade xmlsec to 2.1.8 -area: Authentication -type: enhancement -issues: [] diff --git a/docs/changelog/97775.yaml b/docs/changelog/97775.yaml deleted file mode 100644 index 9d6eee2569c31..0000000000000 --- a/docs/changelog/97775.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97775 -summary: Fix NPE in Desired Balance API -area: Allocation -type: bug -issues: [] diff --git a/docs/changelog/97840.yaml b/docs/changelog/97840.yaml deleted file mode 100644 index 3bcf62328c873..0000000000000 --- a/docs/changelog/97840.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97840 -summary: Improve exception handling in Coordinator#publish -area: Cluster Coordination -type: bug -issues: - - 97798 diff --git a/docs/changelog/97936.yaml b/docs/changelog/97936.yaml deleted file mode 100644 index 2cf0c3e381b94..0000000000000 --- a/docs/changelog/97936.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97936 -summary: Migrate to data tiers routing configures correct default for mounted indices -area: ILM+SLM -type: bug -issues: - - 97898 diff --git a/docs/changelog/97979.yaml b/docs/changelog/97979.yaml deleted file mode 100644 index e7907a761ebf7..0000000000000 --- a/docs/changelog/97979.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 97979 -summary: Ignore the `total_shards_per_node` setting on searchable snapshots in frozen -area: ILM+SLM -type: bug -issues: [] diff --git a/docs/changelog/97987.yaml b/docs/changelog/97987.yaml deleted file mode 100644 index a44d219e37769..0000000000000 --- a/docs/changelog/97987.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 97987 -summary: '`UnmappedFieldFetcher` should ignore nested fields' -area: Search -type: bug -issues: - - 97684 diff --git a/docs/changelog/98091.yaml b/docs/changelog/98091.yaml deleted file mode 100644 index 2042babd3b090..0000000000000 --- a/docs/changelog/98091.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 98091 -summary: '`GlobalAggregator` should call rewrite() before `createWeight()`' -area: Aggregations -type: bug -issues: - - 98076 diff --git a/docs/changelog/98113.yaml b/docs/changelog/98113.yaml deleted file mode 100644 index ceb081ea5b11d..0000000000000 --- a/docs/changelog/98113.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 98113 -summary: Fix APM trace start time -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/98167.yaml b/docs/changelog/98167.yaml deleted file mode 100644 index 4622c9fc2b037..0000000000000 --- a/docs/changelog/98167.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 98167 -summary: Fix failure processing Question Answering model output where the input has been spanned over multiple sequences -area: Machine Learning -type: bug -issues: - - 97917 diff --git a/docs/changelog/98176.yaml b/docs/changelog/98176.yaml deleted file mode 100644 index 938166b5c2d7d..0000000000000 --- a/docs/changelog/98176.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 98176 -summary: Enhance regex performance with duplicate wildcards -area: Infra/Core -type: enhancement -issues: [] diff --git a/docs/changelog/98296.yaml b/docs/changelog/98296.yaml deleted file mode 100644 index 94c5fb0e8ab4b..0000000000000 --- a/docs/changelog/98296.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 98296 -summary: Add setting to scale the processor count used in the model assignment planner -area: Machine Learning -type: enhancement -issues: [] From 805372c2759ccfcbf6d6f754f5cc4dde3b540b0a Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 17 Aug 2023 12:59:02 -0400 Subject: [PATCH 06/12] [buildkite] Send Slack notifications to #lucene for lucene_snapshot failures --- catalog-info.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/catalog-info.yaml b/catalog-info.yaml index 4f5d06357b312..2258cc11beeef 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -160,6 +160,10 @@ spec: spec: repository: elastic/elasticsearch pipeline_file: .buildkite/pipelines/lucene-snapshot/build-snapshot.yml + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true" + SLACK_NOTIFICATIONS_CHANNEL: "#lucene" + SLACK_NOTIFICATIONS_ALL_BRANCHES: "true" branch_configuration: lucene_snapshot default_branch: lucene_snapshot teams: @@ -200,6 +204,10 @@ spec: spec: repository: elastic/elasticsearch pipeline_file: .buildkite/pipelines/lucene-snapshot/update-branch.yml + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true" + SLACK_NOTIFICATIONS_CHANNEL: "#lucene" + SLACK_NOTIFICATIONS_ALL_BRANCHES: "true" branch_configuration: lucene_snapshot default_branch: lucene_snapshot teams: @@ -240,6 +248,10 @@ spec: spec: repository: elastic/elasticsearch pipeline_file: .buildkite/pipelines/lucene-snapshot/run-tests.yml + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true" + SLACK_NOTIFICATIONS_CHANNEL: "#lucene" + SLACK_NOTIFICATIONS_ALL_BRANCHES: "true" branch_configuration: lucene_snapshot default_branch: lucene_snapshot teams: From 44e61341f29e030e8bbcd5477287d66fafca13f3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 17 Aug 2023 13:51:44 -0400 Subject: [PATCH 07/12] ESQL: COALESCE function (#98542) This adds a `COALESCE` function that returns the first non-null value. --- docs/reference/esql/esql-functions.asciidoc | 2 + docs/reference/esql/esql-syntax.asciidoc | 16 +- .../esql/functions/coalesce.asciidoc | 13 ++ .../src/main/resources/conditional.csv-spec | 58 ------ .../src/main/resources/null.csv-spec | 137 ++++++++++++++ .../src/main/resources/show.csv-spec | 1 + .../function/EsqlFunctionRegistry.java | 3 + .../function/scalar/conditional/Case.java | 10 +- .../function/scalar/nulls/Coalesce.java | 162 ++++++++++++++++ .../xpack/esql/io/stream/PlanNamedTypes.java | 10 + .../function/AbstractFunctionTestCase.java | 8 +- .../function/scalar/nulls/CoalesceTests.java | 176 ++++++++++++++++++ .../IsNotNullTests.java | 2 +- .../{conditional => nulls}/IsNullTests.java | 2 +- 14 files changed, 532 insertions(+), 68 deletions(-) create mode 100644 docs/reference/esql/functions/coalesce.asciidoc create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/{conditional => nulls}/IsNotNullTests.java (99%) rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/{conditional => nulls}/IsNullTests.java (99%) diff --git a/docs/reference/esql/esql-functions.asciidoc b/docs/reference/esql/esql-functions.asciidoc index 27581bd6ae57d..226d3e09734b9 100644 --- a/docs/reference/esql/esql-functions.asciidoc +++ b/docs/reference/esql/esql-functions.asciidoc @@ -16,6 +16,7 @@ these functions: * <> * <> * <> +* <> * <> * <> * <> @@ -73,6 +74,7 @@ include::functions/atan2.asciidoc[] include::functions/auto_bucket.asciidoc[] include::functions/case.asciidoc[] include::functions/cidr_match.asciidoc[] +include::functions/coalesce.asciidoc[] include::functions/concat.asciidoc[] include::functions/cos.asciidoc[] include::functions/cosh.asciidoc[] diff --git a/docs/reference/esql/esql-syntax.asciidoc b/docs/reference/esql/esql-syntax.asciidoc index 53d80b9e29ab7..e98f47586e345 100644 --- a/docs/reference/esql/esql-syntax.asciidoc +++ b/docs/reference/esql/esql-syntax.asciidoc @@ -120,15 +120,23 @@ The following boolean operators are supported: For NULL comparison use the `IS NULL` and `IS NOT NULL` predicates: -[source,esql] +[source.merge.styled,esql] ---- -include::{esql-specs}/conditional.csv-spec[tag=is-null] +include::{esql-specs}/null.csv-spec[tag=is-null] ---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/null.csv-spec[tag=is-not-null-result] +|=== -[source,esql] +[source.merge.styled,esql] ---- -include::{esql-specs}/conditional.csv-spec[tag=is-not-null] +include::{esql-specs}/null.csv-spec[tag=is-not-null] ---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/null.csv-spec[tag=is-not-null-result] +|=== [discrete] [[esql-timespan-literals]] diff --git a/docs/reference/esql/functions/coalesce.asciidoc b/docs/reference/esql/functions/coalesce.asciidoc new file mode 100644 index 0000000000000..1fadc20741ec5 --- /dev/null +++ b/docs/reference/esql/functions/coalesce.asciidoc @@ -0,0 +1,13 @@ +[[esql-coalesce]] +=== `COALESCE` + +Returns the first non-null value. + +[source.merge.styled,esql] +---- +include::{esql-specs}/null.csv-spec[tag=coalesce] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/null.csv-spec[tag=coalesce-result] +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec index 7bc5f4b13ae0e..7ea510ccb8b92 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec @@ -89,61 +89,3 @@ M |10 M |10 M |10 ; - -isNull -from employees -| where gender is null -| sort first_name -| keep first_name, gender -| limit 3; - -first_name:keyword|gender:keyword -Berni |null -Cristinel |null -Duangkaew |null -; - -notIsNull -from employees -| where gender is not null -| sort first_name -| keep first_name, gender -| limit 3; - -first_name:keyword|gender:keyword -Alejandro |F -Amabile |M -Anneke |F -; - -isNullForDocs -// tag::is-null[] -FROM employees -| WHERE birth_date IS NULL -| KEEP first_name, last_name -| SORT first_name -| LIMIT 3 -// end::is-null[] -; - -// tag::is-null-result[] -first_name:keyword|last_name:keyword -Basil |Tramer -Florian |Syrotiuk -Lucien |Rosenbaum -// end::is-null-result[] -; - -isNotNullForDocs -// tag::is-not-null[] -FROM employees -| WHERE is_rehired IS NOT NULL -| STATS count(emp_no) -// end::is-not-null[] -; - -// tag::is-not-null-result[] -count(emp_no):long -84 -// end::is-not-null-result[] -; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec new file mode 100644 index 0000000000000..9159dc048b717 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec @@ -0,0 +1,137 @@ +isNull +from employees +| where gender is null +| sort first_name +| keep first_name, gender +| limit 3; + +first_name:keyword|gender:keyword +Berni |null +Cristinel |null +Duangkaew |null +; + +notIsNull +from employees +| where gender is not null +| sort first_name +| keep first_name, gender +| limit 3; + +first_name:keyword|gender:keyword +Alejandro |F +Amabile |M +Anneke |F +; + +isNullForDocs +// tag::is-null[] +FROM employees +| WHERE birth_date IS NULL +| KEEP first_name, last_name +| SORT first_name +| LIMIT 3 +// end::is-null[] +; + +// tag::is-null-result[] +first_name:keyword|last_name:keyword +Basil |Tramer +Florian |Syrotiuk +Lucien |Rosenbaum +// end::is-null-result[] +; + +isNotNullForDocs +// tag::is-not-null[] +FROM employees +| WHERE is_rehired IS NOT NULL +| STATS count(emp_no) +// end::is-not-null[] +; + +// tag::is-not-null-result[] +count(emp_no):long +84 +// end::is-not-null-result[] +; + +coalesceSimple +// tag::coalesce[] +ROW a=null, b="b" +| EVAL COALESCE(a, b) +// end::coalesce[] +; + +// tag::coalesce-result[] +a:null | b:keyword | COALESCE(a,b):keyword + null | b | b +// end::coalesce-result[] +; + +coalesce +FROM employees +| EVAL first_name = COALESCE(first_name, "X") +| SORT first_name DESC, emp_no ASC +| KEEP emp_no, first_name +| limit 10; + +emp_no:integer | first_name:keyword + 10047 | Zvonko + 10081 | Zhongwei + 10026 | Yongqiao + 10043 | Yishay + 10050 | Yinghua + 10087 | Xinglin + 10030 | X + 10031 | X + 10032 | X + 10033 | X +; + +coalesceBackwards +FROM employees +| EVAL first_name = COALESCE("X", first_name) +| SORT first_name DESC, emp_no ASC +| KEEP emp_no, first_name +| limit 10; + +emp_no:integer | first_name:keyword + 10001 | X + 10002 | X + 10003 | X + 10004 | X + 10005 | X + 10006 | X + 10007 | X + 10008 | X + 10009 | X + 10010 | X +; + +coalesceEndsInNull +# ending in null is sill because it'll noop but it shouldn't break things. +FROM employees +| EVAL first_name = COALESCE(first_name, last_name, null) +| SORT first_name DESC, emp_no ASC +| KEEP emp_no, first_name +| limit 3; + +emp_no:integer | first_name:keyword + 10047 | Zvonko + 10081 | Zhongwei + 10026 | Yongqiao +; + +coalesceFolding +FROM employees +| EVAL foo=COALESCE(true, false, null) +| SORT emp_no ASC +| KEEP emp_no, first_name, foo +| limit 3; + +emp_no:integer | first_name:keyword | foo:boolean + 10001 | Georgi | true + 10002 | Bezalel | true + 10003 | Parto | true +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index 1f7bdd3fc6957..e8bc505ff2aed 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -18,6 +18,7 @@ auto_bucket |auto_bucket(arg1, arg2, arg3, arg4) avg |avg(arg1) case |case(arg1...) cidr_match |cidr_match(arg1, arg2...) +coalesce |coalesce(arg1...) concat |concat(arg1, arg2...) cos |cos(arg1) cosh |cosh(arg1) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 1d1a1f4591056..900aaf99a1bc7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -65,6 +65,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMedian; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; @@ -144,6 +145,8 @@ private FunctionDefinition[][] functions() { def(Now.class, Now::new, "now") }, // conditional new FunctionDefinition[] { def(Case.class, Case::new, "case") }, + // null + new FunctionDefinition[] { def(Coalesce.class, Coalesce::new, "coalesce"), }, // IP new FunctionDefinition[] { def(CIDRMatch.class, CIDRMatch::new, "cidr_match") }, // conversion functions diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index 691c49dea7506..780ff99eeec94 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -181,7 +181,15 @@ private record CaseEvaluator(ElementType resultType, List co EvalOperator.ExpressionEvaluator { @Override public Block eval(Page page) { - // Evaluate row at a time for now because its simpler. Much slower. But simpler. + /* + * We have to evaluate lazily so any errors or warnings that would be + * produced by the right hand side are avoided. And so if anything + * on the right hand side is slow we skip it. + * + * And it'd be good if that lazy evaluation were fast. But this + * implementation isn't. It's fairly simple - running position at + * a time - but it's not at all fast. + */ int positionCount = page.getPositionCount(); Block.Builder result = resultType.newBlockBuilder(positionCount); position: for (int p = 0; p < positionCount; p++) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java new file mode 100644 index 0000000000000..4bc6e9246a4ed --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner; +import org.elasticsearch.xpack.esql.planner.Mappable; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Nullability; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; + +/** + * Function returning the first non-null value. + */ +public class Coalesce extends ScalarFunction implements Mappable { + private DataType dataType; + + public Coalesce(Source source, List expressions) { + super(source, expressions); + } + + @Override + public DataType dataType() { + if (dataType == null) { + resolveType(); + } + return dataType; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + for (int position = 0; position < children().size(); position++) { + if (dataType == null || dataType == NULL) { + dataType = children().get(position).dataType(); + continue; + } + TypeResolution resolution = TypeResolutions.isType( + children().get(position), + t -> t == dataType, + sourceText(), + TypeResolutions.ParamOrdinal.fromIndex(position), + dataType.typeName() + ); + if (resolution.unresolved()) { + return resolution; + } + } + return TypeResolution.TYPE_RESOLVED; + } + + @Override + public Nullability nullable() { + // If any of the children aren't nullable then this isn't. + for (Expression c : children()) { + if (c.nullable() == Nullability.FALSE) { + return Nullability.FALSE; + } + } + /* + * Otherwise let's call this one "unknown". If we returned TRUE here + * an optimizer rule would replace this with null if any of our children + * fold to null. We don't want that at all. + */ + return Nullability.UNKNOWN; + } + + @Override + public ScriptTemplate asScript() { + throw new UnsupportedOperationException(); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Coalesce(source(), newChildren); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Coalesce::new, children()); + } + + @Override + public boolean foldable() { + return Expressions.foldable(children()); + } + + @Override + public Object fold() { + return Mappable.super.fold(); + } + + @Override + public Supplier toEvaluator( + Function> toEvaluator + ) { + List> evaluatorSuppliers = children().stream().map(toEvaluator).toList(); + return () -> new CoalesceEvaluator( + LocalExecutionPlanner.toElementType(dataType()), + evaluatorSuppliers.stream().map(Supplier::get).toList() + ); + } + + private record CoalesceEvaluator(ElementType resultType, List evaluators) + implements + EvalOperator.ExpressionEvaluator { + @Override + public Block eval(Page page) { + /* + * We have to evaluate lazily so any errors or warnings that would be + * produced by the right hand side are avoided. And so if anything + * on the right hand side is slow we skip it. + * + * And it'd be good if that lazy evaluation were fast. But this + * implementation isn't. It's fairly simple - running position at + * a time - but it's not at all fast. + */ + int positionCount = page.getPositionCount(); + Block.Builder result = resultType.newBlockBuilder(positionCount); + position: for (int p = 0; p < positionCount; p++) { + int[] positions = new int[] { p }; + Page limited = new Page( + 1, + IntStream.range(0, page.getBlockCount()).mapToObj(b -> page.getBlock(b).filter(positions)).toArray(Block[]::new) + ); + for (EvalOperator.ExpressionEvaluator eval : evaluators) { + Block e = eval.eval(limited); + if (false == e.isNull(0)) { + result.copyFrom(e, 0, 1); + continue position; + } + } + result.appendNull(); + } + return result.build(); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index f94f3677fe3cc..65f35178c2699 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -76,6 +76,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMedian; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; @@ -325,6 +326,7 @@ public static List namedTypeEntries() { of(ScalarFunction.class, AutoBucket.class, PlanNamedTypes::writeAutoBucket, PlanNamedTypes::readAutoBucket), of(ScalarFunction.class, Case.class, PlanNamedTypes::writeCase, PlanNamedTypes::readCase), of(ScalarFunction.class, CIDRMatch.class, PlanNamedTypes::writeCIDRMatch, PlanNamedTypes::readCIDRMatch), + of(ScalarFunction.class, Coalesce.class, PlanNamedTypes::writeCoalesce, PlanNamedTypes::readCoalesce), of(ScalarFunction.class, Concat.class, PlanNamedTypes::writeConcat, PlanNamedTypes::readConcat), of(ScalarFunction.class, DateExtract.class, PlanNamedTypes::writeDateExtract, PlanNamedTypes::readDateExtract), of(ScalarFunction.class, DateFormat.class, PlanNamedTypes::writeDateFormat, PlanNamedTypes::readDateFormat), @@ -1132,6 +1134,14 @@ static void writeCase(PlanStreamOutput out, Case caseValue) throws IOException { out.writeCollection(caseValue.children(), writerFromPlanWriter(PlanStreamOutput::writeExpression)); } + static Coalesce readCoalesce(PlanStreamInput in) throws IOException { + return new Coalesce(Source.EMPTY, in.readList(readerFromPlanReader(PlanStreamInput::readExpression))); + } + + static void writeCoalesce(PlanStreamOutput out, Coalesce coalesce) throws IOException { + out.writeCollection(coalesce.children(), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + static Concat readConcat(PlanStreamInput in) throws IOException { return new Concat(Source.EMPTY, in.readExpression(), in.readList(readerFromPlanReader(PlanStreamInput::readExpression))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 60da1359ebc7e..723bd8f5896c8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -201,8 +201,6 @@ protected final Supplier evaluator(Expression e = new Literal(e.source(), e.fold(), e.dataType()); } Layout.Builder builder = new Layout.Builder(); - // Hack together a layout by scanning for Fields. - // Those will show up in the layout in whatever order a depth first traversal finds them. buildLayout(builder, e); return EvalMapper.toEvaluator(e, builder.build()); } @@ -211,7 +209,11 @@ protected final Page row(List values) { return new Page(BlockUtils.fromListRow(values)); } - private void buildLayout(Layout.Builder builder, Expression e) { + /** + * Hack together a layout by scanning for Fields. + * Those will show up in the layout in whatever order a depth first traversal finds them. + */ + protected void buildLayout(Layout.Builder builder, Expression e) { if (e instanceof FieldAttribute f) { builder.appendChannel(f.id()); return; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java new file mode 100644 index 0000000000000..20543516060d7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.planner.EvalMapper; +import org.elasticsearch.xpack.esql.planner.Layout; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.Nullability; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.type.EsField; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; +import static org.hamcrest.Matchers.equalTo; + +public class CoalesceTests extends AbstractFunctionTestCase { + private static final int MAX_WIDTH = 10; + + public CoalesceTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + /** + * Generate the test cases for this test. The tests don't actually include + * any nulls, but we insert those nulls in {@link #testSimpleWithNulls()}. + */ + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + kwds(suppliers); + longs(suppliers); + ints(suppliers); + return parameterSuppliersFromTypedData(suppliers); + } + + private static void kwds(List suppliers) { + for (int i = 1; i < MAX_WIDTH; i++) { + suppliers.add(kwds(IntStream.range(0, i).mapToObj(v -> "v" + v).toArray(String[]::new))); + } + } + + private static TestCaseSupplier kwds(String... data) { + int[] i = new int[1]; + List typedData = Arrays.stream(data) + .map(d -> new TypedData(new BytesRef(d), DataTypes.KEYWORD, "field" + i[0]++)) + .toList(); + return new TestCaseSupplier(testCaseName(Arrays.stream(data)), () -> testCase(typedData, DataTypes.KEYWORD, ElementType.BYTES_REF)); + } + + private static void longs(List suppliers) { + for (int i = 1; i < MAX_WIDTH; i++) { + suppliers.add(longs(IntStream.range(0, i).mapToObj(v -> Long.valueOf(v)).toArray(Long[]::new))); + } + } + + private static TestCaseSupplier longs(Long... data) { + int[] i = new int[1]; + List typedData = Arrays.stream(data).map(d -> new TypedData(d, DataTypes.LONG, "field" + i[0]++)).toList(); + return new TestCaseSupplier(testCaseName(Arrays.stream(data)), () -> testCase(typedData, DataTypes.LONG, ElementType.LONG)); + } + + private static void ints(List suppliers) { + for (int i = 1; i < MAX_WIDTH; i++) { + suppliers.add(ints(IntStream.range(0, i).mapToObj(v -> Integer.valueOf(v)).toArray(Integer[]::new))); + } + } + + private static TestCaseSupplier ints(Integer... data) { + int[] i = new int[1]; + List typedData = Arrays.stream(data).map(d -> new TypedData(d, DataTypes.INTEGER, "field" + i[0]++)).toList(); + return new TestCaseSupplier(testCaseName(Arrays.stream(data)), () -> testCase(typedData, DataTypes.INTEGER, ElementType.INT)); + } + + private static String testCaseName(Stream data) { + return "(" + data.map(Objects::toString).collect(Collectors.joining(", ")) + ")"; + } + + private static TestCase testCase(List typedData, DataType expectedType, ElementType expectedElementType) { + return new TestCase( + typedData, + expectedToString(expectedElementType, typedData.size()), + expectedType, + equalTo(typedData.stream().map(d -> d.data()).filter(d -> d != null).findFirst().orElse(null)) + ); + } + + private static String expectedToString(ElementType resultType, int attrs) { + return "CoalesceEvaluator[resultType=" + + resultType + + ", evaluators=[" + + IntStream.range(0, attrs).mapToObj(i -> "Attribute[channel=" + i + "]").collect(Collectors.joining(", ")) + + "]]"; + } + + @Override + protected void assertSimpleWithNulls(List data, Block value, int nullBlock) { + for (int i = 0; i < data.size(); i++) { + if (nullBlock == i) { + continue; + } + Object v = data.get(i); + if (v == null) { + continue; + } + assertThat(toJavaObject(value, 0), equalTo(v)); + return; + } + assertThat(value.isNull(0), equalTo(true)); + } + + @Override + protected Coalesce build(Source source, List args) { + return new Coalesce(Source.EMPTY, args.stream().toList()); + } + + public void testCoalesceIsLazy() { + List sub = new ArrayList<>(testCase.getDataAsFields()); + FieldAttribute evil = new FieldAttribute(Source.EMPTY, "evil", new EsField("evil", sub.get(0).dataType(), Map.of(), true)); + sub.add(evil); + Coalesce exp = build(Source.EMPTY, sub); + Layout.Builder builder = new Layout.Builder(); + buildLayout(builder, exp); + Layout layout = builder.build(); + assertThat(toJavaObject(exp.toEvaluator(child -> { + if (child == evil) { + return () -> page -> { throw new AssertionError("shouldn't be called"); }; + } + return EvalMapper.toEvaluator(child, layout); + }).get().eval(row(testCase.getDataValues())), 0), testCase.getMatcher()); + } + + public void testCoalesceNullabilityIsUnknown() { + assertThat(buildFieldExpression(testCase).nullable(), equalTo(Nullability.UNKNOWN)); + } + + public void testCoalesceKnownNullable() { + List sub = new ArrayList<>(testCase.getDataAsFields()); + sub.add(between(0, sub.size()), new Literal(Source.EMPTY, null, sub.get(0).dataType())); + Coalesce exp = build(Source.EMPTY, sub); + // Still UNKNOWN - if it were TRUE then an optimizer would replace it with null + assertThat(exp.nullable(), equalTo(Nullability.UNKNOWN)); + } + + public void testCoalesceNotNullable() { + List sub = new ArrayList<>(testCase.getDataAsFields()); + sub.add(between(0, sub.size()), randomLiteral(sub.get(0).dataType())); + Coalesce exp = build(Source.EMPTY, sub); + // Known not to be nullable because it contains a non-null literal + assertThat(exp.nullable(), equalTo(Nullability.FALSE)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/IsNotNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java similarity index 99% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/IsNotNullTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java index cf12f3f130c9e..12e1d8a680052 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/IsNotNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/IsNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java similarity index 99% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/IsNullTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java index e8d218119d3c1..799471a799cf9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/IsNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; From 29dec0d26dca093a3a5f1366be3114cd071b564d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 17 Aug 2023 21:08:23 +0200 Subject: [PATCH 08/12] reserve a new transportversion in order to change wire serialization on a (#98392) autoscaling API for the serverless project --- server/src/main/java/org/elasticsearch/TransportVersion.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index 4897e009db8d0..c8ca78388ff14 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -176,6 +176,7 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId // 8.10.0 release version is: public static final TransportVersion V_8_500_061 = registerTransportVersion(8_500_061, "4e07f830-8be4-448c-851e-62b3d2f0bf0a"); public static final TransportVersion V_8_500_062 = registerTransportVersion(8_500_062, "09CD9C9B-3207-4B40-8756-B7A12001A885"); + public static final TransportVersion V_8_500_063 = registerTransportVersion(8_500_063, "31dedced-0055-4f34-b952-2f6919be7488"); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ @@ -198,7 +199,7 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId */ private static class CurrentHolder { - private static final TransportVersion CURRENT = findCurrent(V_8_500_062); + private static final TransportVersion CURRENT = findCurrent(V_8_500_063); // finds the pluggable current version, or uses the given fallback private static TransportVersion findCurrent(TransportVersion fallback) { From 34c5bbc52deed9b25a0379f8ac6d595d440e33d3 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 17 Aug 2023 16:08:55 -0400 Subject: [PATCH 09/12] Fixing dense vectors bwc tests to reflect new default version (#98621) New defaults were NOT added for 8.10. They were added for 8.11. closes: https://github.com/elastic/elasticsearch/issues/98611 relates: https://github.com/elastic/elasticsearch/pull/98268 relates: https://github.com/elastic/elasticsearch/pull/97092 --- .../search.vectors/80_dense_vector_indexed_by_default.yml | 4 ++-- .../resources/rest-api-spec/test/search/330_fetch_fields.yml | 4 ++-- .../index/mapper/vectors/DenseVectorFieldMapper.java | 2 +- .../index/mapper/vectors/DenseVectorFieldMapperTests.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index fb6460f971aae..7f67d53f31384 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -1,7 +1,7 @@ setup: - skip: - version: ' - 8.9.99' - reason: 'dense_vector indexed by default was added in 8.10' + version: ' - 8.10.99' + reason: 'dense_vector indexed by default was added in 8.11' --- "Indexed by default with cosine similarity": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index c29d0b3dcd945..03a2129c76598 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1124,8 +1124,8 @@ fetch geo_point: --- "Test with subobjects: false": - skip: - version: ' - 8.9.1' - reason: 'https://github.com/elastic/elasticsearch/issues/96700 fixed after 8.9.0' + version: ' - 8.9.99' + reason: 'https://github.com/elastic/elasticsearch/issues/96700 fixed in 8.10.0' - do: indices.create: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 434fddc951bda..7976940e50858 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -69,7 +69,7 @@ */ public class DenseVectorFieldMapper extends FieldMapper { public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersion.V_7_5_0; - public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersion.V_8_10_0; + public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersion.V_8_11_0; public static final IndexVersion LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION = IndexVersion.V_8_9_0; public static final String CONTENT_TYPE = "dense_vector"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 70f76b5edd3f3..9fc812028609b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -61,7 +61,7 @@ public class DenseVectorFieldMapperTests extends MapperTestCase { - private static final IndexVersion INDEXED_BY_DEFAULT_PREVIOUS_INDEX_VERSION = IndexVersion.V_8_9_1; + private static final IndexVersion INDEXED_BY_DEFAULT_PREVIOUS_INDEX_VERSION = IndexVersion.V_8_10_0; private final ElementType elementType; private final boolean indexed; private final boolean indexOptionsSet; From b3b382ab764c355e09758831b92406b0919e18e9 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 17 Aug 2023 16:10:30 -0400 Subject: [PATCH 10/12] Muting test (#98627) relates: https://github.com/elastic/elasticsearch/issues/98626 --- .../elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 64c216aff28e8..78a89877cbfef 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -260,6 +260,7 @@ protected JobUpdate doParseInstance(XContentParser parser) { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98626") public void testMergeWithJob() { List detectorUpdates = new ArrayList<>(); List detectionRules1 = Collections.singletonList( From e2b0d7bff8fe2635591cf0cbcf45244e01d58cc6 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 17 Aug 2023 14:06:33 -0700 Subject: [PATCH 11/12] Avoid exceeded semaphore permits in ESQL tests (#98625) Closes #98603 --- .../org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index 500017e73e615..f9be6c86bb591 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -197,7 +197,7 @@ private void cancelTask(TaskId taskId) { CancelTasksRequest request = new CancelTasksRequest().setTargetTaskId(taskId).setReason("test cancel"); request.setWaitForCompletion(false); client().admin().cluster().execute(CancelTasksAction.INSTANCE, request).actionGet(); - scriptPermits.release(Integer.MAX_VALUE); + scriptPermits.release(Integer.MAX_VALUE / 2); request = new CancelTasksRequest().setTargetTaskId(taskId).setReason("test cancel"); request.setWaitForCompletion(true); client().admin().cluster().execute(CancelTasksAction.INSTANCE, request).actionGet(); From 6019f38033ccb12bbd3ef01986d59fd6b200ee3d Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 17 Aug 2023 14:07:54 -0700 Subject: [PATCH 12/12] Query ESQL usage stats from new nodes only (#98617) The new ESQL stats action doesn't exist on old nodes; so we should avoid sending stats-requests to those nodes. Closes #98604 --- .../xpack/esql/plugin/TransportEsqlStatsAction.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java index c59792968b274..356c00e52f919 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java @@ -6,9 +6,11 @@ */ package org.elasticsearch.xpack.esql.plugin; +import org.elasticsearch.Version; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; @@ -19,6 +21,7 @@ import org.elasticsearch.xpack.esql.execution.PlanExecutor; import java.io.IOException; +import java.util.Arrays; import java.util.List; /** @@ -54,6 +57,16 @@ public TransportEsqlStatsAction( this.planExecutor = planExecutor; } + @Override + protected void resolveRequest(EsqlStatsRequest request, ClusterState clusterState) { + String[] nodesIds = clusterState.nodes().resolveNodes(request.nodesIds()); + DiscoveryNode[] concreteNodes = Arrays.stream(nodesIds) + .map(clusterState.nodes()::get) + .filter(n -> n.getVersion().onOrAfter(Version.V_8_11_0)) + .toArray(DiscoveryNode[]::new); + request.setConcreteNodes(concreteNodes); + } + @Override protected EsqlStatsResponse newResponse( EsqlStatsRequest request,