Skip to content

Commit

Permalink
ARROW-6580: [Java] Support comparison for unsigned integers
Browse files Browse the repository at this point in the history 
In this issue, we support the comparison of unsigned integer vectors, including UInt1Vector, UInt2Vector, UInt4Vector, and UInt8Vector.
With support for comparison for these vectors, the sort for them is also supported automatically.

Closes #5405 from liyafan82/fly_0917_uint and squashes the following commits:

c94b9b7 <liyafan82>  Add test cases for boundary values
a321784 <liyafan82>  Support comparison for unsigned integers

Authored-by: liyafan82 <[email protected]>
Signed-off-by: Praveen <[email protected]>
  • Loading branch information
@liyafan82 @kszucs
liyafan82 authored and kszucs committed Oct 5, 2019
1 parent c3619b0 commit dd45aea
Show file tree
Hide file tree
Showing 2 changed files with 230 additions and 0 deletions.
86 changes: 86 additions & 0 deletions java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.OFFSET_WIDTH;

import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
import org.apache.arrow.vector.BigIntVector;
Expand All @@ -28,6 +29,10 @@
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.SmallIntVector;
import org.apache.arrow.vector.TinyIntVector;
import org.apache.arrow.vector.UInt1Vector;
import org.apache.arrow.vector.UInt2Vector;
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.UInt8Vector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.BaseRepeatedValueVector;

Expand Down Expand Up @@ -56,6 +61,14 @@ public static <T extends ValueVector> VectorValueComparator<T> createDefaultComp
return (VectorValueComparator<T>) new Float4Comparator();
} else if (vector instanceof Float8Vector) {
return (VectorValueComparator<T>) new Float8Comparator();
} else if (vector instanceof UInt1Vector) {
return (VectorValueComparator<T>) new UInt1Comparator();
} else if (vector instanceof UInt2Vector) {
return (VectorValueComparator<T>) new UInt2Comparator();
} else if (vector instanceof UInt4Vector) {
return (VectorValueComparator<T>) new UInt4Comparator();
} else if (vector instanceof UInt8Vector) {
return (VectorValueComparator<T>) new UInt8Comparator();
}
} else if (vector instanceof BaseVariableWidthVector) {
return (VectorValueComparator<T>) new VariableWidthComparator();
Expand Down Expand Up @@ -141,6 +154,79 @@ public int compareNotNull(int index1, int index2) {
}
}

/**
* Default comparator for unsigned bytes.
* The comparison is based on values, with null comes first.
*/
public static class UInt1Comparator extends VectorValueComparator<UInt1Vector> {

public UInt1Comparator() {
super(1);
}

@Override
public int compareNotNull(int index1, int index2) {
byte value1 = vector1.get(index1);
byte value2 = vector2.get(index2);

return (value1 & 0xff) - (value2 & 0xff);
}
}

/**
* Default comparator for unsigned short integer.
* The comparison is based on values, with null comes first.
*/
public static class UInt2Comparator extends VectorValueComparator<UInt2Vector> {

public UInt2Comparator() {
super(2);
}

@Override
public int compareNotNull(int index1, int index2) {
char value1 = vector1.get(index1);
char value2 = vector2.get(index2);
return value1 - value2;
}
}

/**
* Default comparator for unsigned integer.
* The comparison is based on values, with null comes first.
*/
public static class UInt4Comparator extends VectorValueComparator<UInt4Vector> {

public UInt4Comparator() {
super(4);
}

@Override
public int compareNotNull(int index1, int index2) {
int value1 = vector1.get(index1);
int value2 = vector2.get(index2);
return ByteFunctionHelpers.unsignedIntCompare(value1, value2);
}
}

/**
* Default comparator for unsigned long integer.
* The comparison is based on values, with null comes first.
*/
public static class UInt8Comparator extends VectorValueComparator<UInt8Vector> {

public UInt8Comparator() {
super(8);
}

@Override
public int compareNotNull(int index1, int index2) {
long value1 = vector1.get(index1);
long value2 = vector2.get(index2);
return ByteFunctionHelpers.unsignedLongCompare(value1, value2);
}
}

/**
* Default comparator for float type.
* The comparison is based on values, with null comes first.
Expand Down
144 changes: 144 additions & 0 deletions .../algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.UInt1Vector;
import org.apache.arrow.vector.UInt2Vector;
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.UInt8Vector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.pojo.FieldType;
Expand Down Expand Up @@ -107,4 +111,144 @@ public void testCompareLists() {
assertTrue(comparator.compare(0, 0) == 0);
}
}

@Test
public void testCompareUInt1() {
try (UInt1Vector vec = new UInt1Vector("", allocator)) {
vec.allocateNew(10);
vec.setValueCount(10);

vec.setNull(0);
vec.set(1, -2);
vec.set(2, -1);
vec.set(3, 0);
vec.set(4, 1);
vec.set(5, 2);
vec.set(6, -2);
vec.setNull(7);
vec.set(8, Byte.MAX_VALUE);
vec.set(9, Byte.MIN_VALUE);

VectorValueComparator<UInt1Vector> comparator =
DefaultVectorComparators.createDefaultComparator(vec);
comparator.attachVector(vec);

assertTrue(comparator.compare(0, 1) < 0);
assertTrue(comparator.compare(1, 2) < 0);
assertTrue(comparator.compare(1, 3) > 0);
assertTrue(comparator.compare(2, 5) > 0);
assertTrue(comparator.compare(4, 5) < 0);
assertTrue(comparator.compare(1, 6) == 0);
assertTrue(comparator.compare(0, 7) == 0);
assertTrue(comparator.compare(8, 9) < 0);
assertTrue(comparator.compare(4, 8) < 0);
assertTrue(comparator.compare(5, 9) < 0);
assertTrue(comparator.compare(2, 9) > 0);
}
}

@Test
public void testCompareUInt2() {
try (UInt2Vector vec = new UInt2Vector("", allocator)) {
vec.allocateNew(10);
vec.setValueCount(10);

vec.setNull(0);
vec.set(1, -2);
vec.set(2, -1);
vec.set(3, 0);
vec.set(4, 1);
vec.set(5, 2);
vec.set(6, -2);
vec.setNull(7);
vec.set(8, Short.MAX_VALUE);
vec.set(9, Short.MIN_VALUE);

VectorValueComparator<UInt2Vector> comparator =
DefaultVectorComparators.createDefaultComparator(vec);
comparator.attachVector(vec);

assertTrue(comparator.compare(0, 1) < 0);
assertTrue(comparator.compare(1, 2) < 0);
assertTrue(comparator.compare(1, 3) > 0);
assertTrue(comparator.compare(2, 5) > 0);
assertTrue(comparator.compare(4, 5) < 0);
assertTrue(comparator.compare(1, 6) == 0);
assertTrue(comparator.compare(0, 7) == 0);
assertTrue(comparator.compare(8, 9) < 0);
assertTrue(comparator.compare(4, 8) < 0);
assertTrue(comparator.compare(5, 9) < 0);
assertTrue(comparator.compare(2, 9) > 0);
}
}

@Test
public void testCompareUInt4() {
try (UInt4Vector vec = new UInt4Vector("", allocator)) {
vec.allocateNew(10);
vec.setValueCount(10);

vec.setNull(0);
vec.set(1, -2);
vec.set(2, -1);
vec.set(3, 0);
vec.set(4, 1);
vec.set(5, 2);
vec.set(6, -2);
vec.setNull(7);
vec.set(8, Integer.MAX_VALUE);
vec.set(9, Integer.MIN_VALUE);

VectorValueComparator<UInt4Vector> comparator =
DefaultVectorComparators.createDefaultComparator(vec);
comparator.attachVector(vec);

assertTrue(comparator.compare(0, 1) < 0);
assertTrue(comparator.compare(1, 2) < 0);
assertTrue(comparator.compare(1, 3) > 0);
assertTrue(comparator.compare(2, 5) > 0);
assertTrue(comparator.compare(4, 5) < 0);
assertTrue(comparator.compare(1, 6) == 0);
assertTrue(comparator.compare(0, 7) == 0);
assertTrue(comparator.compare(8, 9) < 0);
assertTrue(comparator.compare(4, 8) < 0);
assertTrue(comparator.compare(5, 9) < 0);
assertTrue(comparator.compare(2, 9) > 0);
}
}

@Test
public void testCompareUInt8() {
try (UInt8Vector vec = new UInt8Vector("", allocator)) {
vec.allocateNew(10);
vec.setValueCount(10);

vec.setNull(0);
vec.set(1, -2);
vec.set(2, -1);
vec.set(3, 0);
vec.set(4, 1);
vec.set(5, 2);
vec.set(6, -2);
vec.setNull(7);
vec.set(8, Long.MAX_VALUE);
vec.set(9, Long.MIN_VALUE);

VectorValueComparator<UInt8Vector> comparator =
DefaultVectorComparators.createDefaultComparator(vec);
comparator.attachVector(vec);

assertTrue(comparator.compare(0, 1) < 0);
assertTrue(comparator.compare(1, 2) < 0);
assertTrue(comparator.compare(1, 3) > 0);
assertTrue(comparator.compare(2, 5) > 0);
assertTrue(comparator.compare(4, 5) < 0);
assertTrue(comparator.compare(1, 6) == 0);
assertTrue(comparator.compare(0, 7) == 0);
assertTrue(comparator.compare(8, 9) < 0);
assertTrue(comparator.compare(4, 8) < 0);
assertTrue(comparator.compare(5, 9) < 0);
assertTrue(comparator.compare(2, 9) > 0);
}
}
}

0 comments on commit dd45aea

Please sign in to comment.