From ad8a09d06103e66178334338ac99acc758a0108f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 24 Sep 2024 04:29:08 -0700 Subject: [PATCH] various fixes and improvements to vectorization fallback (#17098) changes: * add `ApplyFunction` support to vectorization fallback, allowing many of the remaining expressions to be vectorized * add `CastToObjectVectorProcessor` so that vector engine can correctly cast any type * add support for array and complex vector constants * reduce number of cases which can block vectorization in expression planner to be unknown inputs (such as unknown multi-valuedness) * fix array constructor expression, apply map expression to make actual evaluated type match the output type inference * fix bug in array_contains where something like array_contains([null], 'hello') would return true if the array was a numeric array since the non-null string value would cast to a null numeric * fix isNull/isNotNull to correctly handle any type of input argument --- .../apache/druid/math/expr/ApplyFunction.java | 27 +++- .../apache/druid/math/expr/ConstantExpr.java | 19 +-- .../org/apache/druid/math/expr/Function.java | 49 +++--- .../druid/math/expr/FunctionalExpr.java | 141 ++-------------- .../apache/druid/math/expr/LambdaExpr.java | 151 ++++++++++++++++++ .../vector/CastToObjectVectorProcessor.java | 61 +++++++ .../vector/CastToStringVectorProcessor.java | 8 +- .../vector/CastToTypeVectorProcessor.java | 22 +-- ...tDoubleInFunctionVectorValueProcessor.java | 2 +- ...bleLongInFunctionVectorValueProcessor.java | 4 +- ...DoublesInFunctionVectorValueProcessor.java | 4 +- ...gDoubleInFunctionVectorValueProcessor.java | 4 +- ...OutLongInFunctionVectorValueProcessor.java | 2 +- ...utLongsInFunctionVectorValueProcessor.java | 4 +- .../expr/vector/ExprEvalObjectVector.java | 2 +- .../expr/vector/FallbackVectorProcessor.java | 21 +++ ...tDoubleInFunctionVectorValueProcessor.java | 2 +- ...bleLongInFunctionVectorValueProcessor.java | 4 +- ...DoublesInFunctionVectorValueProcessor.java | 4 +- ...gDoubleInFunctionVectorValueProcessor.java | 4 +- ...OutLongInFunctionVectorValueProcessor.java | 2 +- ...utLongsInFunctionVectorValueProcessor.java | 4 +- ...ongOutObjectInFunctionVectorProcessor.java | 2 +- ...ngOutObjectsInFunctionVectorProcessor.java | 4 +- ...ctOutObjectsInFunctionVectorProcessor.java | 4 +- .../math/expr/vector/VectorProcessors.java | 141 ++++++++-------- .../expr/vector/VectorStringProcessors.java | 3 +- .../expression/TimestampFloorExprMacro.java | 2 +- .../expression/TimestampShiftExprMacro.java | 6 +- .../segment/virtual/ExpressionPlanner.java | 10 +- .../virtual/ExpressionVectorSelectors.java | 6 +- .../apache/druid/math/expr/FunctionTest.java | 4 +- .../druid/math/expr/VectorExprSanityTest.java | 65 ++++++-- .../virtual/ExpressionPlannerTest.java | 24 +-- .../sql/calcite/CalciteArraysQueryTest.java | 21 --- .../CalciteMultiValueStringQueryTest.java | 21 --- .../druid/sql/calcite/CalciteQueryTest.java | 21 --- 37 files changed, 487 insertions(+), 388 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/math/expr/LambdaExpr.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/CastToObjectVectorProcessor.java diff --git a/processing/src/main/java/org/apache/druid/math/expr/ApplyFunction.java b/processing/src/main/java/org/apache/druid/math/expr/ApplyFunction.java index e2b1537fb0ab..475432c19f68 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ApplyFunction.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ApplyFunction.java @@ -23,6 +23,7 @@ import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.UOE; import org.apache.druid.math.expr.vector.ExprVectorProcessor; @@ -135,10 +136,28 @@ ExprEval applyMap(@Nullable ExpressionType arrayType, LambdaExpr expr, Indexable { final int length = bindings.getLength(); Object[] out = new Object[length]; + final boolean computeArrayType = arrayType == null; + ExpressionType arrayElementType = arrayType != null + ? (ExpressionType) arrayType.getElementType() + : null; + final ExprEval[] outEval = computeArrayType ? new ExprEval[length] : null; for (int i = 0; i < length; i++) { - - ExprEval evaluated = expr.eval(bindings.withIndex(i)); - arrayType = Function.ArrayConstructorFunction.setArrayOutput(arrayType, out, i, evaluated); + final ExprEval eval = expr.eval(bindings.withIndex(i)); + if (computeArrayType && outEval[i].value() != null) { + arrayElementType = ExpressionTypeConversion.leastRestrictiveType(arrayElementType, eval.type()); + outEval[i] = eval; + } else { + out[i] = eval.castTo(arrayElementType).value(); + } + } + if (arrayElementType == null) { + arrayElementType = NullHandling.sqlCompatible() ? ExpressionType.LONG : ExpressionType.STRING; + } + if (computeArrayType) { + arrayType = ExpressionTypeFactory.getInstance().ofArray(arrayElementType); + for (int i = 0; i < length; i++) { + out[i] = outEval[i].castTo(arrayElementType).value(); + } } return ExprEval.ofArray(arrayType, out); } @@ -237,7 +256,7 @@ public ExprEval apply(LambdaExpr lambdaExpr, List argsExpr, Expr.ObjectBin List> product = CartesianList.create(arrayInputs); CartesianMapLambdaBinding lambdaBinding = new CartesianMapLambdaBinding(elementType, product, lambdaExpr, bindings); ExpressionType lambdaType = lambdaExpr.getOutputType(lambdaBinding); - return applyMap(ExpressionType.asArrayType(lambdaType), lambdaExpr, lambdaBinding); + return applyMap(lambdaType == null ? null : ExpressionTypeFactory.getInstance().ofArray(lambdaType), lambdaExpr, lambdaBinding); } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java b/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java index f6007512f6c7..8f708c95361b 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java @@ -127,6 +127,11 @@ public Expr asSingleThreaded(InputBindingInspector inspector) return new ExprEvalBasedConstantExpr(realEval()); } + @Override + public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector inspector) + { + return VectorProcessors.constant(value, inspector.getMaxVectorSize(), outputType); + } /** * Constant expression based on a concreate ExprEval. * @@ -415,7 +420,7 @@ protected ExprEval realEval() @Override public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector inspector) { - return VectorProcessors.constant(value, inspector.getMaxVectorSize()); + return VectorProcessors.constant(value, inspector.getMaxVectorSize(), ExpressionType.STRING); } @Override @@ -459,12 +464,6 @@ protected ExprEval realEval() return ExprEval.ofArray(outputType, value); } - @Override - public boolean canVectorize(InputBindingInspector inspector) - { - return false; - } - @Override public String stringify() { @@ -547,12 +546,6 @@ protected ExprEval realEval() return ExprEval.ofComplex(outputType, value); } - @Override - public boolean canVectorize(InputBindingInspector inspector) - { - return false; - } - @Override public String stringify() { diff --git a/processing/src/main/java/org/apache/druid/math/expr/Function.java b/processing/src/main/java/org/apache/druid/math/expr/Function.java index 3f0a6e7aa9b2..f79a0fc5a743 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/Function.java +++ b/processing/src/main/java/org/apache/druid/math/expr/Function.java @@ -2026,7 +2026,8 @@ public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingInspe { return CastToTypeVectorProcessor.cast( args.get(0).asVectorProcessor(inspector), - ExpressionType.fromString(StringUtils.toUpperCase(args.get(1).getLiteralValue().toString())) + ExpressionType.fromString(StringUtils.toUpperCase(args.get(1).getLiteralValue().toString())), + inspector.getMaxVectorSize() ); } } @@ -3357,19 +3358,24 @@ public String name() @Override public ExprEval apply(List args, Expr.ObjectBinding bindings) { - // this is copied from 'BaseMapFunction.applyMap', need to find a better way to consolidate, or construct arrays, - // or.. something... final int length = args.size(); Object[] out = new Object[length]; - ExpressionType arrayType = null; - + ExpressionType arrayElementType = null; + final ExprEval[] outEval = new ExprEval[length]; for (int i = 0; i < length; i++) { - ExprEval evaluated = args.get(i).eval(bindings); - arrayType = setArrayOutput(arrayType, out, i, evaluated); + outEval[i] = args.get(i).eval(bindings); + if (outEval[i].value() != null) { + arrayElementType = ExpressionTypeConversion.leastRestrictiveType(arrayElementType, outEval[i].type()); + } } - - return ExprEval.ofArray(arrayType, out); + if (arrayElementType == null) { + arrayElementType = NullHandling.sqlCompatible() ? ExpressionType.LONG : ExpressionType.STRING; + } + for (int i = 0; i < length; i++) { + out[i] = outEval[i].castTo(arrayElementType).value(); + } + return ExprEval.ofArray(ExpressionTypeFactory.getInstance().ofArray(arrayElementType), out); } @Override @@ -3394,28 +3400,6 @@ public ExpressionType getOutputType(Expr.InputBindingInspector inspector, List args, Expr.ObjectBinding bindings) return ExprEval.ofLongBoolean(Arrays.asList(array1).containsAll(Arrays.asList(array2))); } else { final Object elem = rhsExpr.castTo((ExpressionType) array1Type.getElementType()).value(); + if (elem == null && rhsExpr.value() != null) { + return ExprEval.ofLongBoolean(false); + } return ExprEval.ofLongBoolean(Arrays.asList(array1).contains(elem)); } } diff --git a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java index 64dd9f78c208..a7319e0f0287 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java +++ b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java @@ -19,7 +19,6 @@ package org.apache.druid.math.expr; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.StringUtils; @@ -30,8 +29,6 @@ import javax.annotation.Nullable; import java.util.List; import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; @SuppressWarnings("unused") final class FunctionalExpr @@ -39,127 +36,6 @@ final class FunctionalExpr // phony class to enable maven to track the compilation of this class } -@SuppressWarnings("ClassName") -class LambdaExpr implements Expr -{ - private final ImmutableList args; - private final Expr expr; - - LambdaExpr(List args, Expr expr) - { - this.args = ImmutableList.copyOf(args); - this.expr = expr; - } - - @Override - public String toString() - { - return StringUtils.format("(%s -> %s)", args, expr); - } - - int identifierCount() - { - return args.size(); - } - - @Nullable - public String getIdentifier() - { - Preconditions.checkState(args.size() < 2, "LambdaExpr has multiple arguments, use getIdentifiers"); - if (args.size() == 1) { - return args.get(0).toString(); - } - return null; - } - - public List getIdentifiers() - { - return args.stream().map(IdentifierExpr::toString).collect(Collectors.toList()); - } - - public List stringifyIdentifiers() - { - return args.stream().map(IdentifierExpr::stringify).collect(Collectors.toList()); - } - - ImmutableList getIdentifierExprs() - { - return args; - } - - public Expr getExpr() - { - return expr; - } - - @Override - public boolean canVectorize(InputBindingInspector inspector) - { - return expr.canVectorize(inspector); - } - - @Override - public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector inspector) - { - return expr.asVectorProcessor(inspector); - } - - @Override - public ExprEval eval(ObjectBinding bindings) - { - return expr.eval(bindings); - } - - @Override - public String stringify() - { - return StringUtils.format("(%s) -> %s", ARG_JOINER.join(stringifyIdentifiers()), expr.stringify()); - } - - @Override - public Expr visit(Shuttle shuttle) - { - List newArgs = - args.stream().map(arg -> (IdentifierExpr) shuttle.visit(arg)).collect(Collectors.toList()); - Expr newBody = expr.visit(shuttle); - return shuttle.visit(new LambdaExpr(newArgs, newBody)); - } - - @Override - public BindingAnalysis analyzeInputs() - { - final Set lambdaArgs = args.stream().map(IdentifierExpr::toString).collect(Collectors.toSet()); - BindingAnalysis bodyDetails = expr.analyzeInputs(); - return bodyDetails.removeLambdaArguments(lambdaArgs); - } - - @Override - public ExpressionType getOutputType(InputBindingInspector inspector) - { - return expr.getOutputType(inspector); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - LambdaExpr that = (LambdaExpr) o; - return Objects.equals(args, that.args) && - Objects.equals(expr, that.expr); - } - - @Override - public int hashCode() - { - return Objects.hash(args, expr); - } -} - /** * {@link Expr} node for a {@link Function} call. {@link FunctionExpr} has children {@link Expr} in the form of the * list of arguments that are passed to the {@link Function} along with the {@link Expr.ObjectBinding} when it is @@ -350,15 +226,24 @@ public ExprEval eval(ObjectBinding bindings) @Override public boolean canVectorize(InputBindingInspector inspector) { - return function.canVectorize(inspector, lambdaExpr, argsExpr) && - lambdaExpr.canVectorize(inspector) && - argsExpr.stream().allMatch(expr -> expr.canVectorize(inspector)); + return canVectorizeNative(inspector) || (getOutputType(inspector) != null && inspector.canVectorize(argsExpr)); } @Override public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector inspector) { - return function.asVectorProcessor(inspector, lambdaExpr, argsExpr); + if (canVectorizeNative(inspector)) { + return function.asVectorProcessor(inspector, lambdaExpr, argsExpr); + } else { + return FallbackVectorProcessor.create(function, lambdaExpr, argsExpr, inspector); + } + } + + private boolean canVectorizeNative(InputBindingInspector inspector) + { + return function.canVectorize(inspector, lambdaExpr, argsExpr) && + lambdaExpr.canVectorize(inspector) && + argsExpr.stream().allMatch(expr -> expr.canVectorize(inspector)); } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/LambdaExpr.java b/processing/src/main/java/org/apache/druid/math/expr/LambdaExpr.java new file mode 100644 index 000000000000..2dc39dc24d18 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/LambdaExpr.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class LambdaExpr implements Expr +{ + private final ImmutableList args; + private final Expr expr; + + LambdaExpr(List args, Expr expr) + { + this.args = ImmutableList.copyOf(args); + this.expr = expr; + } + + @Override + public String toString() + { + return StringUtils.format("(%s -> %s)", args, expr); + } + + int identifierCount() + { + return args.size(); + } + + @Nullable + public String getIdentifier() + { + Preconditions.checkState(args.size() < 2, "LambdaExpr has multiple arguments, use getIdentifiers"); + if (args.size() == 1) { + return args.get(0).toString(); + } + return null; + } + + public List getIdentifiers() + { + return args.stream().map(IdentifierExpr::toString).collect(Collectors.toList()); + } + + public List stringifyIdentifiers() + { + return args.stream().map(IdentifierExpr::stringify).collect(Collectors.toList()); + } + + ImmutableList getIdentifierExprs() + { + return args; + } + + public Expr getExpr() + { + return expr; + } + + @Override + public boolean canVectorize(InputBindingInspector inspector) + { + return expr.canVectorize(inspector); + } + + @Override + public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector inspector) + { + return expr.asVectorProcessor(inspector); + } + + @Override + public ExprEval eval(ObjectBinding bindings) + { + return expr.eval(bindings); + } + + @Override + public String stringify() + { + return StringUtils.format("(%s) -> %s", ARG_JOINER.join(stringifyIdentifiers()), expr.stringify()); + } + + @Override + public Expr visit(Shuttle shuttle) + { + List newArgs = + args.stream().map(arg -> (IdentifierExpr) shuttle.visit(arg)).collect(Collectors.toList()); + Expr newBody = expr.visit(shuttle); + return shuttle.visit(new LambdaExpr(newArgs, newBody)); + } + + @Override + public BindingAnalysis analyzeInputs() + { + final Set lambdaArgs = args.stream().map(IdentifierExpr::toString).collect(Collectors.toSet()); + BindingAnalysis bodyDetails = expr.analyzeInputs(); + return bodyDetails.removeLambdaArguments(lambdaArgs); + } + + @Override + public ExpressionType getOutputType(InputBindingInspector inspector) + { + return expr.getOutputType(inspector); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LambdaExpr that = (LambdaExpr) o; + return Objects.equals(args, that.args) && + Objects.equals(expr, that.expr); + } + + @Override + public int hashCode() + { + return Objects.hash(args, expr); + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/CastToObjectVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToObjectVectorProcessor.java new file mode 100644 index 000000000000..32d72d4b4bcd --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToObjectVectorProcessor.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExpressionType; + +public class CastToObjectVectorProcessor extends CastToTypeVectorProcessor +{ + private final ExpressionType outputType; + private final ExpressionType delegateType; + private final Object[] output; + + public CastToObjectVectorProcessor( + ExprVectorProcessor delegate, + ExpressionType outputType, + int maxVectorSize + ) + { + super(delegate); + this.delegateType = delegate.getOutputType(); + this.outputType = outputType; + this.output = new Object[maxVectorSize]; + } + + @Override + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) + { + final ExprEvalVector delegateOutput = delegate.evalVector(bindings); + final Object[] toCast = delegateOutput.getObjectVector(); + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { + ExprEval cast = ExprEval.ofType(delegateType, toCast[i]).castTo(outputType); + output[i] = cast.value(); + } + return new ExprEvalObjectVector(output, outputType); + } + + @Override + public ExpressionType getOutputType() + { + return outputType; + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java index f0e2dc7ee42a..e3d5f06f8a12 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java @@ -25,9 +25,12 @@ public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor { - public CastToStringVectorProcessor(ExprVectorProcessor delegate) + private final Object[] output; + + public CastToStringVectorProcessor(ExprVectorProcessor delegate, int maxVectorSize) { super(delegate); + this.output = new Object[maxVectorSize]; } @Override @@ -35,8 +38,7 @@ public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { ExprEvalVector result = delegate.evalVector(bindings); final Object[] objects = result.getObjectVector(); - final Object[] output = new String[objects.length]; - for (int i = 0; i < objects.length; i++) { + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { output[i] = Evals.asString(objects[i]); } return new ExprEvalObjectVector(output, ExpressionType.STRING); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java index eff7633cfbab..c6448b67ed20 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java @@ -20,7 +20,6 @@ package org.apache.druid.math.expr.vector; import org.apache.druid.math.expr.ExpressionType; -import org.apache.druid.math.expr.Exprs; public abstract class CastToTypeVectorProcessor implements ExprVectorProcessor { @@ -31,24 +30,29 @@ protected CastToTypeVectorProcessor(ExprVectorProcessor delegate) this.delegate = delegate; } - public static ExprVectorProcessor cast(ExprVectorProcessor delegate, ExpressionType type) + public static ExprVectorProcessor cast( + ExprVectorProcessor castInput, + ExpressionType castToType, + int maxVectorSize + ) { final ExprVectorProcessor caster; - if (delegate.getOutputType().equals(type)) { - caster = delegate; + final ExpressionType inputType = castInput.getOutputType(); + if (inputType.equals(castToType)) { + caster = castInput; } else { - switch (type.getType()) { + switch (castToType.getType()) { case STRING: - caster = new CastToStringVectorProcessor(delegate); + caster = new CastToStringVectorProcessor(castInput, maxVectorSize); break; case LONG: - caster = new CastToLongVectorProcessor(delegate); + caster = new CastToLongVectorProcessor(castInput); break; case DOUBLE: - caster = new CastToDoubleVectorProcessor(delegate); + caster = new CastToDoubleVectorProcessor(castInput); break; default: - throw Exprs.cannotVectorize(); + caster = new CastToObjectVectorProcessor(castInput, castToType, maxVectorSize); } } return (ExprVectorProcessor) caster; diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorValueProcessor.java index 578daf0e4886..8edcae53c36b 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorValueProcessor.java @@ -29,7 +29,7 @@ public abstract class DoubleOutDoubleInFunctionVectorValueProcessor { public DoubleOutDoubleInFunctionVectorValueProcessor(ExprVectorProcessor processor, int maxVectorSize) { - super(CastToTypeVectorProcessor.cast(processor, ExpressionType.DOUBLE), maxVectorSize); + super(CastToTypeVectorProcessor.cast(processor, ExpressionType.DOUBLE, maxVectorSize), maxVectorSize); } public abstract double apply(double input); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorValueProcessor.java index aac396aa7da5..ae3a0a5d8da1 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public DoubleOutDoubleLongInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE), - CastToTypeVectorProcessor.cast(right, ExpressionType.LONG), + CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.LONG, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorValueProcessor.java index a3a7d77b1da5..8033b267cae6 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public DoubleOutDoublesInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE), - CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE), + CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorValueProcessor.java index dc62fe1b027d..b98004d4774a 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public DoubleOutLongDoubleInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.LONG), - CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE), + CastToTypeVectorProcessor.cast(left, ExpressionType.LONG, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorValueProcessor.java index 0b833d02089a..123ebcd67abc 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorValueProcessor.java @@ -29,7 +29,7 @@ public abstract class DoubleOutLongInFunctionVectorValueProcessor { public DoubleOutLongInFunctionVectorValueProcessor(ExprVectorProcessor processor, int maxVectorSize) { - super(CastToTypeVectorProcessor.cast(processor, ExpressionType.LONG), maxVectorSize); + super(CastToTypeVectorProcessor.cast(processor, ExpressionType.LONG, maxVectorSize), maxVectorSize); } public abstract double apply(long input); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorValueProcessor.java index d9fa3fe2856b..507abf2c0e3f 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public DoubleOutLongsInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.LONG), - CastToTypeVectorProcessor.cast(right, ExpressionType.LONG), + CastToTypeVectorProcessor.cast(left, ExpressionType.LONG, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.LONG, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java index cc22445358fe..05dba92ebd7f 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java @@ -72,7 +72,7 @@ private void computeNumbers() } } else { // ARRAY, COMPLEX - final ExprEval valueEval = ExprEval.bestEffortOf(values[i]); + final ExprEval valueEval = ExprEval.ofType(type, values[i]).castTo(ExpressionType.DOUBLE); longs[i] = valueEval.asLong(); doubles[i] = valueEval.asDouble(); numericNulls[i] = valueEval.isNumericNull(); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java index 99758d0bc6a4..4c6732747528 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java @@ -20,12 +20,14 @@ package org.apache.druid.math.expr.vector; import org.apache.druid.error.DruidException; +import org.apache.druid.math.expr.ApplyFunction; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.Function; +import org.apache.druid.math.expr.LambdaExpr; import javax.annotation.Nullable; import java.util.ArrayList; @@ -73,6 +75,25 @@ public static FallbackVectorProcessor create( ); } + /** + * Create a processor for a non-vectorizable {@link ApplyFunction}. + */ + public static FallbackVectorProcessor create( + final ApplyFunction function, + final LambdaExpr lambdaExpr, + final List args, + final Expr.VectorInputBindingInspector inspector + ) + { + final List adaptedArgs = makeAdaptedArgs(args, inspector); + return makeFallbackProcessor( + () -> function.apply(lambdaExpr, adaptedArgs, UnusedBinding.INSTANCE), + adaptedArgs, + function.getOutputType(inspector, lambdaExpr, args), + inspector + ); + } + /** * Create a processor for a non-vectorizable {@link ExprMacroTable.ExprMacro}. */ diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorValueProcessor.java index 0193e24de21d..13a907faa654 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorValueProcessor.java @@ -29,7 +29,7 @@ public abstract class LongOutDoubleInFunctionVectorValueProcessor { public LongOutDoubleInFunctionVectorValueProcessor(ExprVectorProcessor processor, int maxVectorSize) { - super(CastToTypeVectorProcessor.cast(processor, ExpressionType.DOUBLE), maxVectorSize); + super(CastToTypeVectorProcessor.cast(processor, ExpressionType.DOUBLE, maxVectorSize), maxVectorSize); } public abstract long apply(double input); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleLongInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleLongInFunctionVectorValueProcessor.java index 25e854ee5bd1..f3a58691e687 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleLongInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleLongInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public LongOutDoubleLongInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE), - CastToTypeVectorProcessor.cast(right, ExpressionType.LONG), + CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.LONG, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoublesInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoublesInFunctionVectorValueProcessor.java index bbd6fa181500..cf2193c1a02c 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoublesInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutDoublesInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public LongOutDoublesInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE), - CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE), + CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongDoubleInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongDoubleInFunctionVectorValueProcessor.java index d63ffdd0ce5a..e91213e2396b 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongDoubleInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongDoubleInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public LongOutLongDoubleInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.LONG), - CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE), + CastToTypeVectorProcessor.cast(left, ExpressionType.LONG, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorValueProcessor.java index c13ae84849db..58e80dfc2b31 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorValueProcessor.java @@ -29,7 +29,7 @@ public abstract class LongOutLongInFunctionVectorValueProcessor { public LongOutLongInFunctionVectorValueProcessor(ExprVectorProcessor processor, int maxVectorSize) { - super(CastToTypeVectorProcessor.cast(processor, ExpressionType.LONG), maxVectorSize); + super(CastToTypeVectorProcessor.cast(processor, ExpressionType.LONG, maxVectorSize), maxVectorSize); } public abstract long apply(long input); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorValueProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorValueProcessor.java index 9646ddf711a1..befabea3f1c8 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorValueProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorValueProcessor.java @@ -34,8 +34,8 @@ public LongOutLongsInFunctionVectorValueProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.LONG), - CastToTypeVectorProcessor.cast(right, ExpressionType.LONG), + CastToTypeVectorProcessor.cast(left, ExpressionType.LONG, maxVectorSize), + CastToTypeVectorProcessor.cast(right, ExpressionType.LONG, maxVectorSize), maxVectorSize ); } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java index 7847d71ec224..022eb6d4fe60 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java @@ -33,7 +33,7 @@ public LongOutObjectInFunctionVectorProcessor( ExpressionType inputType ) { - super(CastToTypeVectorProcessor.cast(processor, inputType), maxVectorSize, new long[maxVectorSize]); + super(CastToTypeVectorProcessor.cast(processor, inputType, maxVectorSize), maxVectorSize, new long[maxVectorSize]); } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java index a99342928fb7..89caa4cf0a15 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java @@ -37,8 +37,8 @@ protected LongOutObjectsInFunctionVectorProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, inputType), - CastToTypeVectorProcessor.cast(right, inputType), + CastToTypeVectorProcessor.cast(left, inputType, maxVectorSize), + CastToTypeVectorProcessor.cast(right, inputType, maxVectorSize), new long[maxVectorSize] ); this.outNulls = new boolean[maxVectorSize]; diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java index fc98706fe674..d40ebe5af05a 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java @@ -36,8 +36,8 @@ protected ObjectOutObjectsInFunctionVectorProcessor( ) { super( - CastToTypeVectorProcessor.cast(left, expressionType), - CastToTypeVectorProcessor.cast(right, expressionType), + CastToTypeVectorProcessor.cast(left, expressionType, maxVectorSize), + CastToTypeVectorProcessor.cast(right, expressionType, maxVectorSize), new Object[maxVectorSize] ); this.expressionType = expressionType; diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index f81326438a16..33e4c82cbe8a 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.error.DruidException; import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; @@ -81,15 +82,19 @@ public static ExprVectorProcessor makeSymmetricalProcessor( } /** - * Creates an {@link ExprVectorProcessor} that creates a {@link ExprEvalVector} for a constant {@link String} value. + * Creates an {@link ExprVectorProcessor} that creates a {@link ExprEvalVector} for a constant any non-numeric value. + * Numeric types should use {@link #constant(Double, int)} or {@link #constant(Long, int)} instead. * * @see org.apache.druid.math.expr.ConstantExpr */ - public static ExprVectorProcessor constant(@Nullable String constant, int maxVectorSize) + public static ExprVectorProcessor constant(@Nullable Object constant, int maxVectorSize, ExpressionType type) { - final Object[] strings = new Object[maxVectorSize]; - Arrays.fill(strings, constant); - final ExprEvalObjectVector eval = new ExprEvalObjectVector(strings, ExpressionType.STRING); + if (type.isNumeric()) { + throw DruidException.defensive("Type[%s] should use the numeric constant creator instead", type); + } + final Object[] objects = new Object[maxVectorSize]; + Arrays.fill(objects, constant); + final ExprEvalObjectVector eval = new ExprEvalObjectVector(objects, type); return new ExprVectorProcessor() { @Override @@ -101,7 +106,7 @@ public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) @Override public ExpressionType getOutputType() { - return ExpressionType.STRING; + return type; } }; } @@ -194,15 +199,11 @@ public static ExprVectorProcessor identifier(Expr.VectorInputBindingInspector if (inputType == null) { // nil column, we can be anything, so be a string because it's the most flexible - // (numbers will be populated with default values in default mode and non-null) - return new IdentifierVectorProcessor(ExpressionType.STRING) - { - @Override - public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) - { - return new ExprEvalObjectVector(bindings.getObjectVector(binding), ExpressionType.STRING); - } - }; + if (NullHandling.sqlCompatible()) { + return constant((Long) null, inspector.getMaxVectorSize()); + } else { + return constant(null, inspector.getMaxVectorSize(), ExpressionType.STRING); + } } switch (inputType.getType()) { case LONG: @@ -292,23 +293,27 @@ public static ExprVectorProcessor isNull(Expr.VectorInputBindingInspector } final long[] outputValues = new long[inspector.getMaxVectorSize()]; - ExprVectorProcessor processor = null; - if (Types.is(type, ExprType.STRING)) { - final ExprVectorProcessor input = expr.asVectorProcessor(inspector); + final ExprVectorProcessor processor; + if (Types.is(type, ExprType.LONG)) { + final ExprVectorProcessor input = expr.asVectorProcessor(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); - final Object[] values = inputEval.values(); - for (int i = 0; i < currentSize; i++) { - if (values[i] == null) { - outputValues[i] = 1L; - } else { - outputValues[i] = 0L; + final boolean[] nulls = inputEval.getNullVector(); + if (nulls == null) { + Arrays.fill(outputValues, 0L); + } else { + for (int i = 0; i < currentSize; i++) { + if (nulls[i]) { + outputValues[i] = 1L; + } else { + outputValues[i] = 0L; + } } } return new ExprEvalLongVector(outputValues, null); @@ -320,14 +325,14 @@ public ExpressionType getOutputType() return ExpressionType.LONG; } }; - } else if (Types.is(type, ExprType.LONG)) { - final ExprVectorProcessor input = expr.asVectorProcessor(inspector); + } else if (Types.is(type, ExprType.DOUBLE)) { + final ExprVectorProcessor input = expr.asVectorProcessor(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); final boolean[] nulls = inputEval.getNullVector(); @@ -351,26 +356,22 @@ public ExpressionType getOutputType() return ExpressionType.LONG; } }; - } else if (Types.is(type, ExprType.DOUBLE)) { - final ExprVectorProcessor input = expr.asVectorProcessor(inspector); + } else { + final ExprVectorProcessor input = expr.asVectorProcessor(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); - final boolean[] nulls = inputEval.getNullVector(); - if (nulls == null) { - Arrays.fill(outputValues, 0L); - } else { - for (int i = 0; i < currentSize; i++) { - if (nulls[i]) { - outputValues[i] = 1L; - } else { - outputValues[i] = 0L; - } + final Object[] values = inputEval.values(); + for (int i = 0; i < currentSize; i++) { + if (values[i] == null) { + outputValues[i] = 1L; + } else { + outputValues[i] = 0L; } } return new ExprEvalLongVector(outputValues, null); @@ -384,9 +385,6 @@ public ExpressionType getOutputType() }; } - if (processor == null) { - throw Exprs.cannotVectorize(); - } return (ExprVectorProcessor) processor; } @@ -406,23 +404,27 @@ public static ExprVectorProcessor isNotNull(Expr.VectorInputBindingInspec final long[] outputValues = new long[inspector.getMaxVectorSize()]; - ExprVectorProcessor processor = null; - if (Types.is(type, ExprType.STRING)) { - final ExprVectorProcessor input = expr.asVectorProcessor(inspector); + final ExprVectorProcessor processor; + if (Types.is(type, ExprType.LONG)) { + final ExprVectorProcessor input = expr.asVectorProcessor(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); - final Object[] values = inputEval.values(); - for (int i = 0; i < currentSize; i++) { - if (values[i] == null) { - outputValues[i] = 0L; - } else { - outputValues[i] = 1L; + final boolean[] nulls = inputEval.getNullVector(); + if (nulls == null) { + Arrays.fill(outputValues, 1L); + } else { + for (int i = 0; i < currentSize; i++) { + if (nulls[i]) { + outputValues[i] = 0L; + } else { + outputValues[i] = 1L; + } } } return new ExprEvalLongVector(outputValues, null); @@ -434,14 +436,14 @@ public ExpressionType getOutputType() return ExpressionType.LONG; } }; - } else if (Types.is(type, ExprType.LONG)) { - final ExprVectorProcessor input = expr.asVectorProcessor(inspector); + } else if (Types.is(type, ExprType.DOUBLE)) { + final ExprVectorProcessor input = expr.asVectorProcessor(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); final boolean[] nulls = inputEval.getNullVector(); @@ -465,26 +467,22 @@ public ExpressionType getOutputType() return ExpressionType.LONG; } }; - } else if (Types.is(type, ExprType.DOUBLE)) { - final ExprVectorProcessor input = expr.asVectorProcessor(inspector); + } else { + final ExprVectorProcessor input = expr.asVectorProcessor(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); - final boolean[] nulls = inputEval.getNullVector(); - if (nulls == null) { - Arrays.fill(outputValues, 1L); - } else { - for (int i = 0; i < currentSize; i++) { - if (nulls[i]) { - outputValues[i] = 0L; - } else { - outputValues[i] = 1L; - } + final Object[] values = inputEval.values(); + for (int i = 0; i < currentSize; i++) { + if (values[i] == null) { + outputValues[i] = 0L; + } else { + outputValues[i] = 1L; } } return new ExprEvalLongVector(outputValues, null); @@ -498,9 +496,6 @@ public ExpressionType getOutputType() }; } - if (processor == null) { - throw Exprs.cannotVectorize(); - } return (ExprVectorProcessor) processor; } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java index 3326474f42ed..342c8fa7522f 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java @@ -74,7 +74,8 @@ public static ExprVectorProcessor concat(Expr.VectorInputBindingInspector for (int i = 0; i < inputs.size(); i++) { inputProcessors[i] = CastToTypeVectorProcessor.cast( inputs.get(i).asVectorProcessor(inspector), - ExpressionType.STRING + ExpressionType.STRING, + inspector.getMaxVectorSize() ); } final ExprVectorProcessor processor = new ObjectOutMultiObjectInVectorProcessor( diff --git a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java index 5cd2c88fc803..b2f8f7b82bbd 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java @@ -134,7 +134,7 @@ public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector { ExprVectorProcessor processor; processor = new LongOutLongInFunctionVectorValueProcessor( - CastToTypeVectorProcessor.cast(args.get(0).asVectorProcessor(inspector), ExpressionType.LONG), + CastToTypeVectorProcessor.cast(args.get(0).asVectorProcessor(inspector), ExpressionType.LONG, inspector.getMaxVectorSize()), inspector.getMaxVectorSize() ) { diff --git a/processing/src/main/java/org/apache/druid/query/expression/TimestampShiftExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/TimestampShiftExprMacro.java index 0f820cc6806c..99bf23cc15d8 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/TimestampShiftExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/TimestampShiftExprMacro.java @@ -117,7 +117,11 @@ public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector { ExprVectorProcessor processor; processor = new LongOutLongInFunctionVectorValueProcessor( - CastToTypeVectorProcessor.cast(args.get(0).asVectorProcessor(inspector), ExpressionType.LONG), + CastToTypeVectorProcessor.cast( + args.get(0).asVectorProcessor(inspector), + ExpressionType.LONG, + inspector.getMaxVectorSize() + ), inspector.getMaxVectorSize() ) { diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java index 8e171f36abec..2a68706f421b 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java @@ -182,16 +182,12 @@ public static ExpressionPlan plan(ColumnInspector inspector, Expr expression) traits.remove(ExpressionPlan.Trait.SINGLE_INPUT_MAPPABLE); } - // vectorized expressions do not support incomplete, multi-valued inputs or outputs, or implicit mapping - // they also do not support unknown inputs, but they also do not currently have to deal with them, as missing - // capabilites is indicative of a non-existent column instead of an unknown schema. If this ever changes, + // vectorized expressions do not support unknown inputs, but they also do not currently have to deal with them, as + // missing capabilites is indicative of a non-existent column instead of an unknown schema. If this ever changes, // this check should also change boolean supportsVector = ExpressionPlan.none( traits, - ExpressionPlan.Trait.INCOMPLETE_INPUTS, - ExpressionPlan.Trait.NEEDS_APPLIED, - ExpressionPlan.Trait.NON_SCALAR_INPUTS, - ExpressionPlan.Trait.NON_SCALAR_OUTPUT + ExpressionPlan.Trait.INCOMPLETE_INPUTS ); if (supportsVector && expression.canVectorize(inspector)) { diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java index 5c022cbf3055..8578be228d59 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java @@ -170,7 +170,8 @@ public static VectorObjectSelector castValueSelectorToObject( return new ExpressionVectorObjectSelector( CastToTypeVectorProcessor.cast( VectorProcessors.identifier(binding, columnName), - ExpressionType.fromColumnType(castTo) + ExpressionType.fromColumnType(castTo), + binding.getMaxVectorSize() ), binding ); @@ -190,7 +191,8 @@ public static VectorValueSelector castObjectSelectorToNumeric( return new ExpressionVectorValueSelector( CastToTypeVectorProcessor.cast( VectorProcessors.identifier(binding, columnName), - ExpressionType.fromColumnType(castTo) + ExpressionType.fromColumnType(castTo), + binding.getMaxVectorSize() ), binding ); diff --git a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java index bc74227e5f4e..40aa6a30ac41 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -309,7 +309,7 @@ public void testRpad() public void testArrayConstructor() { assertArrayExpr("array(1, 2, 3, 4)", new Long[]{1L, 2L, 3L, 4L}); - assertArrayExpr("array(1, 2, 3, 'bar')", new Long[]{1L, 2L, 3L, null}); + assertArrayExpr("array(1, 2, 3, 'bar')", new String[]{"1", "2", "3", "bar"}); assertArrayExpr("array(1.0)", new Double[]{1.0}); assertArrayExpr("array('foo', 'bar')", new String[]{"foo", "bar"}); assertArrayExpr( @@ -1218,7 +1218,7 @@ public void testArrayToMultiValueStringWithValidInputs() assertArrayExpr("array_to_mv(a)", new String[]{"foo", "bar", "baz", "foobar"}); assertArrayExpr("array_to_mv(b)", new String[]{"1", "2", "3", "4", "5"}); assertArrayExpr("array_to_mv(c)", new String[]{"3.1", "4.2", "5.3"}); - assertArrayExpr("array_to_mv(array(y,z))", new String[]{"2", "3"}); + assertArrayExpr("array_to_mv(array(y,z))", new String[]{"2.0", "3.1"}); // array type is determined by the first array type assertArrayExpr("array_to_mv(array_concat(b,c))", new String[]{"1", "2", "3", "4", "5", "3", "4", "5"}); assertArrayExpr( diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index f96c648c9faa..5a32e8f505d1 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -19,6 +19,7 @@ package org.apache.druid.math.expr; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.NonnullPair; @@ -26,6 +27,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.vector.ExprEvalVector; import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.query.expression.NestedDataExpressions; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; @@ -52,6 +54,12 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest private static final int NUM_ITERATIONS = 10; private static final int VECTOR_SIZE = 512; + private static final ExprMacroTable MACRO_TABLE = new ExprMacroTable( + ImmutableList.of( + new NestedDataExpressions.JsonObjectExprMacro() + ) + ); + final Map types = ImmutableMap.builder() .put("l1", ExpressionType.LONG) .put("l2", ExpressionType.LONG) @@ -238,7 +246,7 @@ public void testSymmetricalBivariateFunctions() public void testCast() { final String[] columns = new String[]{"d1", "l1", "s1"}; - final String[] castTo = new String[]{"'STRING'", "'LONG'", "'DOUBLE'"}; + final String[] castTo = new String[]{"'STRING'", "'LONG'", "'DOUBLE'", "'ARRAY'", "'ARRAY'", "'ARRAY'"}; final String[][] args = makeTemplateArgs(columns, castTo); final String[] templates = new String[]{"cast(%s, %s)"}; testFunctions(types, templates, args); @@ -253,6 +261,22 @@ public void testStringFns() testExpression("concat(s1,'-',s2,'-',l1,'-',d1)", types); } + @Test + public void testArrayFns() + { + testExpression("array(s1, s2)", types); + testExpression("array(l1, l2)", types); + testExpression("array(d1, d2)", types); + testExpression("array(l1, d2)", types); + testExpression("array(s1, l2)", types); + } + + @Test + public void testJsonFns() + { + testExpression("json_object('k1', s1, 'k2', l1)", types); + } + @Test public void testConstants() { @@ -263,6 +287,7 @@ public void testConstants() testExpression("Infinity", types); testExpression("-Infinity", types); testExpression("'hello'", types); + testExpression("json_object('a', 1, 'b', 'abc', 'c', 3.3, 'd', array(1,2,3))", types); } static void testFunctions(Map types, String[] templates, String[] args) @@ -288,7 +313,7 @@ static void testFunctions(Map types, String[] templates, static void testExpression(String expr, Map types) { log.debug("[%s]", expr); - Expr parsed = Parser.parse(expr, ExprMacroTable.nil()); + Expr parsed = Parser.parse(expr, MACRO_TABLE); testExpression(expr, parsed, types, NUM_ITERATIONS); testSequentialBinding(expr, parsed, types); @@ -315,11 +340,19 @@ public static void testSequentialBinding( if (outputType != null && !eval.isNumericNull()) { Assert.assertEquals(eval.type(), outputType); } - Assert.assertEquals( - StringUtils.format("Values do not match for row %s for expression %s", i, expr), - eval.valueOrDefault(), - vectorVals[i] - ); + if (outputType != null && outputType.isArray()) { + Assert.assertArrayEquals( + StringUtils.format("Values do not match for row %s for expression %s", i, expr), + (Object[]) eval.valueOrDefault(), + (Object[]) vectorVals[i] + ); + } else { + Assert.assertEquals( + StringUtils.format("Values do not match for row %s for expression %s", i, expr), + eval.valueOrDefault(), + vectorVals[i] + ); + } } } @@ -363,11 +396,19 @@ public ExpressionType getType(String name) if (outputType != null && !eval.isNumericNull()) { Assert.assertEquals(eval.type(), outputType); } - Assert.assertEquals( - StringUtils.format("Values do not match for row %s for expression %s", i, expr), - eval.valueOrDefault(), - vectorVals[i] - ); + if (outputType != null && outputType.isArray()) { + Assert.assertArrayEquals( + StringUtils.format("Values do not match for row %s for expression %s", i, expr), + (Object[]) eval.valueOrDefault(), + (Object[]) vectorVals[i] + ); + } else { + Assert.assertEquals( + StringUtils.format("Values do not match for row %s for expression %s", i, expr), + eval.valueOrDefault(), + vectorVals[i] + ); + } } } } diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java index fcae823b6262..a0f3d1e8ecc4 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java @@ -1151,7 +1151,8 @@ public void testScalarOutputMultiValueInput() Assert.assertTrue( thePlan.is( ExpressionPlan.Trait.NON_SCALAR_INPUTS, - ExpressionPlan.Trait.NEEDS_APPLIED + ExpressionPlan.Trait.NEEDS_APPLIED, + ExpressionPlan.Trait.VECTORIZABLE ) ); Assert.assertFalse( @@ -1160,8 +1161,7 @@ public void testScalarOutputMultiValueInput() ExpressionPlan.Trait.SINGLE_INPUT_MAPPABLE, ExpressionPlan.Trait.NON_SCALAR_OUTPUT, ExpressionPlan.Trait.INCOMPLETE_INPUTS, - ExpressionPlan.Trait.UNKNOWN_INPUTS, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.UNKNOWN_INPUTS ) ); @@ -1251,7 +1251,8 @@ public void testArrayConstruction() ExpressionPlan thePlan = plan("array(long1, long2)"); Assert.assertTrue( thePlan.is( - ExpressionPlan.Trait.NON_SCALAR_OUTPUT + ExpressionPlan.Trait.NON_SCALAR_OUTPUT, + ExpressionPlan.Trait.VECTORIZABLE ) ); Assert.assertFalse( @@ -1261,8 +1262,7 @@ public void testArrayConstruction() ExpressionPlan.Trait.UNKNOWN_INPUTS, ExpressionPlan.Trait.INCOMPLETE_INPUTS, ExpressionPlan.Trait.NEEDS_APPLIED, - ExpressionPlan.Trait.NON_SCALAR_INPUTS, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.NON_SCALAR_INPUTS ) ); Assert.assertEquals(ExpressionType.LONG_ARRAY, thePlan.getOutputType()); @@ -1387,7 +1387,8 @@ private static void assertArrayInput(ExpressionPlan thePlan) { Assert.assertTrue( thePlan.is( - ExpressionPlan.Trait.NON_SCALAR_INPUTS + ExpressionPlan.Trait.NON_SCALAR_INPUTS, + ExpressionPlan.Trait.VECTORIZABLE ) ); Assert.assertFalse( @@ -1397,8 +1398,7 @@ private static void assertArrayInput(ExpressionPlan thePlan) ExpressionPlan.Trait.NON_SCALAR_OUTPUT, ExpressionPlan.Trait.INCOMPLETE_INPUTS, ExpressionPlan.Trait.UNKNOWN_INPUTS, - ExpressionPlan.Trait.NEEDS_APPLIED, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.NEEDS_APPLIED ) ); } @@ -1408,7 +1408,8 @@ private static void assertArrayInAndOut(ExpressionPlan thePlan) Assert.assertTrue( thePlan.is( ExpressionPlan.Trait.NON_SCALAR_INPUTS, - ExpressionPlan.Trait.NON_SCALAR_OUTPUT + ExpressionPlan.Trait.NON_SCALAR_OUTPUT, + ExpressionPlan.Trait.VECTORIZABLE ) ); Assert.assertFalse( @@ -1417,8 +1418,7 @@ private static void assertArrayInAndOut(ExpressionPlan thePlan) ExpressionPlan.Trait.SINGLE_INPUT_MAPPABLE, ExpressionPlan.Trait.INCOMPLETE_INPUTS, ExpressionPlan.Trait.UNKNOWN_INPUTS, - ExpressionPlan.Trait.NEEDS_APPLIED, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.NEEDS_APPLIED ) ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index d3453404cd9d..b2f87b3d6335 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -1525,9 +1525,6 @@ public void testArraySliceArrayColumns() @Test public void testArrayLength() { - // Cannot vectorize due to array expressions. - cannotVectorize(); - testQuery( "SELECT dim1, ARRAY_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC", ImmutableList.of( @@ -1740,9 +1737,6 @@ public void testArrayPrepend() @Test public void testArrayPrependAppend() { - // Cannot vectorize due to array expressions. - cannotVectorize(); - ImmutableList results; if (useDefault) { results = ImmutableList.of( @@ -1861,9 +1855,6 @@ public void testArrayConcat() @Test public void testArrayOffset() { - // Cannot vectorize due to array expressions. - cannotVectorize(); - testQuery( "SELECT ARRAY_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2185,9 +2176,6 @@ public void testArrayGroupAsArrayWithFunction() @Test public void testArrayOrdinal() { - // Cannot vectorize due to array expressions. - cannotVectorize(); - testQuery( "SELECT ARRAY_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2228,9 +2216,6 @@ public void testArrayOrdinal() @Test public void testArrayOffsetOf() { - // Cannot vectorize due to array expressions. - cannotVectorize(); - testQuery( "SELECT ARRAY_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2277,9 +2262,6 @@ public void testArrayOffsetOf() @Test public void testArrayOrdinalOf() { - // Cannot vectorize due to array expressions. - cannotVectorize(); - testQuery( "SELECT ARRAY_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2327,9 +2309,6 @@ public void testArrayOrdinalOf() @Test public void testArrayToString() { - // Cannot vectorize due to array expressions. - cannotVectorize(); - ImmutableList results; if (useDefault) { results = ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java index 8b21d336fdc3..25f9ac34e0db 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java @@ -469,9 +469,6 @@ public void testMultiValueStringSlice() @Test public void testMultiValueStringLength() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT dim1, MV_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC", ImmutableList.of( @@ -632,9 +629,6 @@ public void testMultiValueStringPrepend() @Test public void testMultiValueStringPrependAppend() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - ImmutableList results; if (useDefault) { results = ImmutableList.of( @@ -818,9 +812,6 @@ public void testMultiValueStringConcatBackwardsCompat0dot22andOlder() @Test public void testMultiValueStringOffset() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT MV_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -857,9 +848,6 @@ public void testMultiValueStringOffset() @Test public void testMultiValueStringOrdinal() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT MV_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -900,9 +888,6 @@ public void testMultiValueStringOrdinal() @Test public void testMultiValueStringOffsetOf() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT MV_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -949,9 +934,6 @@ public void testMultiValueStringOffsetOf() @Test public void testMultiValueStringOrdinalOf() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT MV_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -999,9 +981,6 @@ public void testMultiValueStringOrdinalOf() @Test public void testMultiValueStringToString() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - ImmutableList results; if (useDefault) { results = ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index f4c1f1a49b04..55da3f41c23b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -5848,9 +5848,6 @@ public void testInIsNotTrueAndLessThanFilter() @Test public void testInExpression() { - // Cannot vectorize scalar_in_array expression. - cannotVectorize(); - testQuery( "SELECT dim1 IN ('abc', 'def', 'ghi'), COUNT(*)\n" + "FROM druid.foo\n" @@ -5914,9 +5911,6 @@ public void testInExpressionBelowThreshold() @Test public void testInOrIsNullExpression() { - // Cannot vectorize scalar_in_array expression. - cannotVectorize(); - testQuery( "SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL, COUNT(*)\n" + "FROM druid.foo\n" @@ -5948,9 +5942,6 @@ public void testInOrIsNullExpression() @Test public void testNotInOrIsNullExpression() { - // Cannot vectorize scalar_in_array expression. - cannotVectorize(); - testQuery( "SELECT NOT (dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL), COUNT(*)\n" + "FROM druid.foo\n" @@ -5982,9 +5973,6 @@ public void testNotInOrIsNullExpression() @Test public void testNotInAndIsNotNullExpression() { - // Cannot vectorize scalar_in_array expression. - cannotVectorize(); - testQuery( "SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 IS NOT NULL, COUNT(*)\n" + "FROM druid.foo\n" @@ -6016,9 +6004,6 @@ public void testNotInAndIsNotNullExpression() @Test public void testInOrGreaterThanExpression() { - // Cannot vectorize scalar_in_array expression. - cannotVectorize(); - testQuery( "SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 > 'zzz', COUNT(*)\n" + "FROM druid.foo\n" @@ -6050,9 +6035,6 @@ public void testInOrGreaterThanExpression() @Test public void testNotInAndLessThanExpression() { - // Cannot vectorize scalar_in_array expression. - cannotVectorize(); - testQuery( "SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 < 'zzz', COUNT(*)\n" + "FROM druid.foo\n" @@ -6084,9 +6066,6 @@ public void testNotInAndLessThanExpression() @Test public void testNotInOrEqualToOneOfThemExpression() { - // Cannot vectorize scalar_in_array expression. - cannotVectorize(); - testQuery( "SELECT dim1 NOT IN ('abc', 'def', 'ghi') OR dim1 = 'def', COUNT(*)\n" + "FROM druid.foo\n"