diff --git a/presto-docs/src/main/sphinx/functions/array.rst b/presto-docs/src/main/sphinx/functions/array.rst index 45409dbd11b6..973f9807cdf8 100644 --- a/presto-docs/src/main/sphinx/functions/array.rst +++ b/presto-docs/src/main/sphinx/functions/array.rst @@ -62,6 +62,9 @@ Array Functions .. function:: array_except(x, y) -> array Returns an array of elements in ``x`` but not in ``y``, without duplicates. + This function uses ``IS NOT DISTINCT FROM`` to determine which elements are the same. :: + + SELECT array_except(ARRAY[1, 3, 3, 2, null], ARRAY[1,2, 2, 4]) -- ARRAY[3, null] .. function:: array_frequency(array(E)) -> map(E, int) @@ -71,14 +74,24 @@ Array Functions .. function:: array_has_duplicates(array(T)) -> boolean Returns a boolean: whether ``array`` has any elements that occur more than once. + Throws an exception if any of the elements are rows or arrays that contain nulls. :: + + SELECT array_has_duplicates(ARRAY[1, 2, null, 1, null, 3]) -- true + SELECT array_has_duplicates(ARRAY[ROW(1, null), ROW(1, null)]) -- "map key cannot be null or contain nulls" .. function:: array_intersect(x, y) -> array Returns an array of the elements in the intersection of ``x`` and ``y``, without duplicates. + This function uses ``IS NOT DISTINCT FROM`` to determine which elements are the same. :: + + SELECT array_intersect(ARRAY[1, 2, 3, 2, null], ARRAY[1,2, 2, 4, null]) -- ARRAY[1, 2, null] .. function:: array_intersect(array(array(E))) -> array(E) Returns an array of the elements in the intersection of all arrays in the given array, without duplicates. + This function uses ``IS NOT DISTINCT FROM`` to determine which elements are the same. :: + + SELECT array_intersect(ARRAY[ARRAY[1, 2, 3, 2, null], ARRAY[1,2,2, 4, null], ARRAY [1, 2, 3, 4 null]]) -- ARRAY[1, 2, null] .. function:: array_join(x, delimiter, null_replacement) -> varchar @@ -209,6 +222,9 @@ Array Functions .. function:: array_union(x, y) -> array Returns an array of the elements in the union of ``x`` and ``y``, without duplicates. + This function uses ``IS NOT DISTINCT FROM`` to determine which elements are the same. :: + + SELECT array_union(ARRAY[1, 2, 3, 2, null], ARRAY[1,2, 2, 4, null]) -- ARRAY[1, 2, 3, 4 null] .. function:: cardinality(x) -> bigint diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/OptimizedTypedSet.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/OptimizedTypedSet.java index 3d609503752b..907d49d9a603 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/OptimizedTypedSet.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/OptimizedTypedSet.java @@ -21,14 +21,19 @@ import com.facebook.presto.operator.project.SelectedPositions; import org.openjdk.jol.info.ClassLayout; +import java.lang.invoke.MethodHandle; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import static com.facebook.presto.common.array.Arrays.ensureCapacity; +import static com.facebook.presto.common.type.TypeUtils.readNativeValue; import static com.facebook.presto.operator.project.SelectedPositions.positionsList; import static com.facebook.presto.type.TypeUtils.hashPosition; import static com.facebook.presto.type.TypeUtils.positionEqualsPosition; +import static com.facebook.presto.util.Failures.internalError; +import static com.google.common.base.Defaults.defaultValue; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.slice.SizeOf.sizeOf; import static it.unimi.dsi.fastutil.HashCommon.arraySize; @@ -47,6 +52,7 @@ public class OptimizedTypedSet private static final SelectedPositions EMPTY_SELECTED_POSITIONS = positionsList(new int[0], 0, 0); private final Type elementType; + private final Optional elementIsDistinctFrom; private final int hashCapacity; private final int hashMask; @@ -56,17 +62,18 @@ public class OptimizedTypedSet private long[] blockPositionByHash; // Each 64-bit long is 32-bit index for blocks + 32-bit position within block private int currentBlockIndex = -1; // The index into the blocks array and positionsForBlocks list - public OptimizedTypedSet(Type elementType, int maxPositionCount) + public OptimizedTypedSet(Type elementType, MethodHandle elementIsDistinctFrom, int maxPositionCount) { - this(elementType, INITIAL_BLOCK_COUNT, maxPositionCount); + this(elementType, Optional.of(elementIsDistinctFrom), INITIAL_BLOCK_COUNT, maxPositionCount); } - public OptimizedTypedSet(Type elementType, int expectedBlockCount, int maxPositionCount) + public OptimizedTypedSet(Type elementType, Optional elementIsDistinctFrom, int expectedBlockCount, int maxPositionCount) { checkArgument(expectedBlockCount >= 0, "expectedBlockCount must not be negative"); checkArgument(maxPositionCount >= 0, "maxPositionCount must not be negative"); this.elementType = requireNonNull(elementType, "elementType must not be null"); + this.elementIsDistinctFrom = requireNonNull(elementIsDistinctFrom, "elementIsDistinctFrom is null"); this.hashCapacity = arraySize(maxPositionCount, FILL_RATIO); this.hashMask = hashCapacity - 1; @@ -293,7 +300,7 @@ private int getInsertPosition(long[] hashtable, int hashPosition, Block block, i // Already has this element int blockIndex = (int) ((blockPosition & 0xffff_ffff_0000_0000L) >> 32); int positionWithinBlock = (int) (blockPosition & 0xffff_ffff); - if (positionEqualsPosition(elementType, blocks[blockIndex], positionWithinBlock, block, position)) { + if (isContainedAt(blocks[blockIndex], positionWithinBlock, block, position)) { return INVALID_POSITION; } @@ -301,6 +308,23 @@ private int getInsertPosition(long[] hashtable, int hashPosition, Block block, i } } + private boolean isContainedAt(Block firstBlock, int positionWithinFirstBlock, Block secondBlock, int positionWithinSecondBlock) + { + if (elementIsDistinctFrom.isPresent()) { + boolean firstValueNull = firstBlock.isNull(positionWithinFirstBlock); + Object firstValue = firstValueNull ? defaultValue(elementType.getJavaType()) : readNativeValue(elementType, firstBlock, positionWithinFirstBlock); + boolean secondValueNull = secondBlock.isNull(positionWithinSecondBlock); + Object secondValue = secondValueNull ? defaultValue(elementType.getJavaType()) : readNativeValue(elementType, secondBlock, positionWithinSecondBlock); + try { + return !(boolean) elementIsDistinctFrom.get().invoke(firstValue, firstValueNull, secondValue, secondValueNull); + } + catch (Throwable t) { + throw internalError(t); + } + } + return positionEqualsPosition(elementType, firstBlock, positionWithinFirstBlock, secondBlock, positionWithinSecondBlock); + } + /** * Add an element to the hash table if it's not already existed. * @@ -322,7 +346,7 @@ private boolean addElement(long[] hashtable, int hashPosition, Block block, int // Already has this element int blockIndex = (int) ((blockPosition & 0xffff_ffff_0000_0000L) >> 32); int positionWithinBlock = (int) (blockPosition & 0xffff_ffff); - if (positionEqualsPosition(elementType, blocks[blockIndex], positionWithinBlock, block, position)) { + if (isContainedAt(blocks[blockIndex], positionWithinBlock, block, position)) { return false; } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java index ae2ed727da1a..1d891781cc9d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java @@ -17,10 +17,14 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.operator.aggregation.OptimizedTypedSet; import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.OperatorDependency; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; +import java.lang.invoke.MethodHandle; + +import static com.facebook.presto.common.function.OperatorType.IS_DISTINCT_FROM; import static java.lang.Math.max; @ScalarFunction("array_except") @@ -33,6 +37,7 @@ private ArrayExceptFunction() {} @SqlType("array(E)") public static Block except( @TypeParameter("E") Type type, + @OperatorDependency(operator = IS_DISTINCT_FROM, argumentTypes = {"E", "E"}) MethodHandle elementIsDistinctFrom, @SqlType("array(E)") Block leftArray, @SqlType("array(E)") Block rightArray) { @@ -43,7 +48,7 @@ public static Block except( return leftArray; } - OptimizedTypedSet typedSet = new OptimizedTypedSet(type, max(leftPositionCount, rightPositionCount)); + OptimizedTypedSet typedSet = new OptimizedTypedSet(type, elementIsDistinctFrom, max(leftPositionCount, rightPositionCount)); typedSet.union(rightArray); typedSet.except(leftArray); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java index ceb42383e07c..b9c7b1268f30 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java @@ -17,12 +17,17 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.operator.aggregation.OptimizedTypedSet; import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.OperatorDependency; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.SqlInvokedScalarFunction; import com.facebook.presto.spi.function.SqlParameter; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; +import java.lang.invoke.MethodHandle; + +import static com.facebook.presto.common.function.OperatorType.IS_DISTINCT_FROM; + public final class ArrayIntersectFunction { private ArrayIntersectFunction() {} @@ -33,6 +38,7 @@ private ArrayIntersectFunction() {} @SqlType("array(E)") public static Block intersect( @TypeParameter("E") Type type, + @OperatorDependency(operator = IS_DISTINCT_FROM, argumentTypes = {"E", "E"}) MethodHandle elementIsDistinctFrom, @SqlType("array(E)") Block leftArray, @SqlType("array(E)") Block rightArray) { @@ -48,7 +54,7 @@ public static Block intersect( return rightArray; } - OptimizedTypedSet typedSet = new OptimizedTypedSet(type, rightPositionCount); + OptimizedTypedSet typedSet = new OptimizedTypedSet(type, elementIsDistinctFrom, rightPositionCount); typedSet.union(rightArray); typedSet.intersect(leftArray); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayUnionFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayUnionFunction.java index 0a370dcca5b1..f7c16d451deb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayUnionFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayUnionFunction.java @@ -17,10 +17,15 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.operator.aggregation.OptimizedTypedSet; import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.OperatorDependency; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; +import java.lang.invoke.MethodHandle; + +import static com.facebook.presto.common.function.OperatorType.IS_DISTINCT_FROM; + @ScalarFunction("array_union") @Description("Union elements of the two given arrays") public final class ArrayUnionFunction @@ -31,12 +36,13 @@ private ArrayUnionFunction() {} @SqlType("array(E)") public static Block union( @TypeParameter("E") Type type, + @OperatorDependency(operator = IS_DISTINCT_FROM, argumentTypes = {"E", "E"}) MethodHandle elementIsDistinctFrom, @SqlType("array(E)") Block leftArray, @SqlType("array(E)") Block rightArray) { int leftArrayCount = leftArray.getPositionCount(); int rightArrayCount = rightArray.getPositionCount(); - OptimizedTypedSet typedSet = new OptimizedTypedSet(type, leftArrayCount + rightArrayCount); + OptimizedTypedSet typedSet = new OptimizedTypedSet(type, elementIsDistinctFrom, leftArrayCount + rightArrayCount); typedSet.union(leftArray); typedSet.union(rightArray); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java index e0af8cb52261..1e56f3bd5fa6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java @@ -151,7 +151,7 @@ public static Block mapConcat(MapType mapType, Block[] maps) Type valueType = mapType.getValueType(); // We need to divide the entries by 2 because the maps array is SingleMapBlocks and it had the positionCount twice as large as a normal Block - OptimizedTypedSet typedSet = new OptimizedTypedSet(keyType, maps.length, entries / 2); + OptimizedTypedSet typedSet = new OptimizedTypedSet(keyType, Optional.empty(), maps.length, entries / 2); for (int i = lastMapIndex; i >= firstMapIndex; i--) { SingleMapBlock singleMapBlock = (SingleMapBlock) maps[i]; Block keyBlock = singleMapBlock.getKeyBlock(); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestOptimizedTypedSet.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestOptimizedTypedSet.java index 997fbfc5c015..363b77271370 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestOptimizedTypedSet.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestOptimizedTypedSet.java @@ -14,19 +14,24 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.common.block.Block; +import com.facebook.presto.type.BigintOperators; import org.testng.annotations.Test; +import java.lang.invoke.MethodHandle; +import java.util.Optional; + import static com.facebook.presto.block.BlockAssertions.assertBlockEquals; import static com.facebook.presto.block.BlockAssertions.createEmptyBlock; import static com.facebook.presto.block.BlockAssertions.createLongRepeatBlock; import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; +import static com.facebook.presto.common.block.MethodHandleUtil.methodHandle; import static com.facebook.presto.common.type.BigintType.BIGINT; import static org.testng.Assert.fail; public class TestOptimizedTypedSet { - private static final String FUNCTION_NAME = "optimized_typed_set_test"; private static final int POSITIONS_PER_PAGE = 100; + private static final MethodHandle BIGINT_DISTINCT_METHOD_HANDLE = methodHandle(BigintOperators.BigintDistinctFromOperator.class, "isDistinctFrom", long.class, boolean.class, long.class, boolean.class); @Test public void testConstructor() @@ -34,7 +39,7 @@ public void testConstructor() for (int i = -2; i <= -1; i++) { try { //noinspection ResultOfObjectAllocationIgnored - new OptimizedTypedSet(BIGINT, 2, i); + new OptimizedTypedSet(BIGINT, Optional.of(BIGINT_DISTINCT_METHOD_HANDLE), 2, i); fail("Should throw exception if expectedSize < 0"); } catch (IllegalArgumentException e) { @@ -44,7 +49,7 @@ public void testConstructor() try { //noinspection ResultOfObjectAllocationIgnored - new OptimizedTypedSet(null, -1, 1); + new OptimizedTypedSet(null, Optional.of(BIGINT_DISTINCT_METHOD_HANDLE), -1, 1); fail("Should throw exception if expectedBlockCount is negative"); } catch (NullPointerException | IllegalArgumentException e) { @@ -53,7 +58,7 @@ public void testConstructor() try { //noinspection ResultOfObjectAllocationIgnored - new OptimizedTypedSet(null, 2, 1); + new OptimizedTypedSet(null, Optional.of(BIGINT_DISTINCT_METHOD_HANDLE), 2, 1); fail("Should throw exception if type is null"); } catch (NullPointerException | IllegalArgumentException e) { @@ -64,7 +69,7 @@ public void testConstructor() @Test public void testUnionWithDistinctValues() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE + 1); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE + 1); Block block = createLongSequenceBlock(0, POSITIONS_PER_PAGE / 2); testUnion(typedSet, block, block); @@ -80,7 +85,7 @@ public void testUnionWithDistinctValues() @Test public void testUnionWithRepeatingValues() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE); Block block = createLongRepeatBlock(0, POSITIONS_PER_PAGE); Block expectedBlock = createLongRepeatBlock(0, 1); @@ -95,14 +100,14 @@ public void testUnionWithRepeatingValues() @Test public void testIntersectWithEmptySet() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE); testIntersect(typedSet, createLongSequenceBlock(0, POSITIONS_PER_PAGE - 1).appendNull(), createEmptyBlock(BIGINT)); } @Test public void testIntersectWithDistinctValues() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE); Block block = createLongSequenceBlock(0, POSITIONS_PER_PAGE - 1).appendNull(); typedSet.union(block); @@ -119,7 +124,7 @@ public void testIntersectWithDistinctValues() @Test public void testIntersectWithNonDistinctValues() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE); Block block = createLongSequenceBlock(0, POSITIONS_PER_PAGE - 1).appendNull(); typedSet.union(block); @@ -137,7 +142,7 @@ public void testIntersectWithNonDistinctValues() @Test public void testExceptWithDistinctValues() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE); Block block = createLongSequenceBlock(0, POSITIONS_PER_PAGE - 1).appendNull(); typedSet.union(block); @@ -149,7 +154,7 @@ public void testExceptWithDistinctValues() @Test public void testExceptWithRepeatingValues() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE); Block block = createLongRepeatBlock(0, POSITIONS_PER_PAGE - 1).appendNull(); testExcept(typedSet, block, createLongSequenceBlock(0, 1).appendNull()); @@ -158,7 +163,7 @@ public void testExceptWithRepeatingValues() @Test public void testMultipleOperations() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE + 1); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE + 1); Block block = createLongSequenceBlock(0, POSITIONS_PER_PAGE / 2).appendNull(); @@ -176,7 +181,7 @@ public void testMultipleOperations() @Test public void testNulls() { - OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, POSITIONS_PER_PAGE + 1); + OptimizedTypedSet typedSet = new OptimizedTypedSet(BIGINT, BIGINT_DISTINCT_METHOD_HANDLE, POSITIONS_PER_PAGE + 1); // Empty block Block emptyBlock = createLongSequenceBlock(0, 0); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java index c459226f6f0d..96c2db4dabc3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java @@ -14,6 +14,7 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.common.type.RowType; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -23,6 +24,7 @@ import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.common.type.VarcharType.createVarcharType; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; @@ -66,4 +68,31 @@ public void testDuplicates() assertFunction("array_except(ARRAY[CAST('x' as VARCHAR), 'x', 'y', 'z'], ARRAY['x', 'y', 'x'])", new ArrayType(VARCHAR), ImmutableList.of("z")); assertFunction("array_except(ARRAY[true, false, null, true, false, null], ARRAY[true, true, true])", new ArrayType(BOOLEAN), asList(false, null)); } + + @Test + public void testIndeterminateRows() + { + // test unsupported + assertFunction( + "array_except(ARRAY[(123, 'abc'), (123, NULL)], ARRAY[(123, 'abc'), (123, NULL)])", + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of()); + assertFunction( + "array_except(ARRAY[(NULL, 'abc'), (123, null), (123, 'abc')], ARRAY[(456, 'def'),(NULL, 'abc')])", + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of(asList(123, null), asList(123, "abc"))); + } + + @Test + public void testIndeterminateArrays() + { + assertFunction( + "array_except(ARRAY[ARRAY[123, 456], ARRAY[123, NULL]], ARRAY[ARRAY[123, 456], ARRAY[123, NULL]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of()); + assertFunction( + "array_except(ARRAY[ARRAY[NULL, 456], ARRAY[123, null], ARRAY[123, 456]], ARRAY[ARRAY[456, 456],ARRAY[NULL, 456]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of(asList(123, null), asList(123, 456))); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java index 8ec788bb01f9..528eb9521144 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java @@ -126,13 +126,27 @@ public void testRow() @Test public void testIndeterminateRows() { - // test unsupported - assertNotSupported( + assertFunction( "ARRAY_INTERSECT(ARRAY[(123, 'abc'), (123, NULL)], ARRAY[(123, 'abc'), (123, NULL)])", - "ROW comparison not supported for fields with null elements"); - assertNotSupported( + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of(asList(123, "abc"), asList(123, null))); + assertFunction( "ARRAY_INTERSECT(ARRAY[(NULL, 'abc'), (123, 'abc')], ARRAY[(123, 'abc'),(NULL, 'abc')])", - "ROW comparison not supported for fields with null elements"); + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of(asList(null, "abc"), asList(123, "abc"))); + } + + @Test + public void testIndeterminateArrays() + { + assertFunction( + "ARRAY_INTERSECT(ARRAY[ARRAY[123, 456], ARRAY[123, NULL]], ARRAY[ARRAY[123, 456], ARRAY[123, NULL]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of(asList(123, 456), asList(123, null))); + assertFunction( + "ARRAY_INTERSECT(ARRAY[ARRAY[NULL, 456], ARRAY[123, 456]], ARRAY[ARRAY[123, 456],ARRAY[NULL, 456]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of(asList(null, 456), asList(123, 456))); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayUnionFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayUnionFunction.java index 44f659f52733..095a624b32d8 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayUnionFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayUnionFunction.java @@ -15,6 +15,7 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.common.type.RowType; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -69,4 +70,31 @@ public void testNull() assertFunction("ARRAY_UNION(ARRAY [NULL], ARRAY [NULL, NULL])", new ArrayType(UNKNOWN), asList((Object) null)); assertFunction("ARRAY_UNION(ARRAY ['abc', NULL, 'xyz', NULL], ARRAY [NULL, 'abc', NULL, NULL])", new ArrayType(createVarcharType(3)), asList("abc", null, "xyz")); } + + @Test + public void testIndeterminateRows() + { + // test unsupported + assertFunction( + "array_union(ARRAY[(123, 'abc'), (123, NULL)], ARRAY[(123, 'abc'), (123, NULL)])", + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of(asList(123, "abc"), asList(123, null))); + assertFunction( + "array_union(ARRAY[(NULL, 'abc'), (123, null), (123, 'abc')], ARRAY[(456, 'def'),(NULL, 'abc')])", + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of(asList(null, "abc"), asList(123, null), asList(123, "abc"), asList(456, "def"))); + } + + @Test + public void testIndeterminateArrays() + { + assertFunction( + "array_union(ARRAY[ARRAY[123, 456], ARRAY[123, NULL]], ARRAY[ARRAY[123, 456], ARRAY[123, NULL]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of(asList(123, 456), asList(123, null))); + assertFunction( + "array_union(ARRAY[ARRAY[NULL, 456], ARRAY[123, 456]], ARRAY[ARRAY[123, 456],ARRAY[NULL, 456]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of(asList(null, 456), asList(123, 456))); + } }