Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HBASE-28025 Enhance ByteBufferUtils.findCommonPrefix to compare 8 bytes each time #5354

Merged
merged 1 commit into from
Aug 20, 2023
Merged

HBASE-28025 Enhance ByteBufferUtils.findCommonPrefix to compare 8 bytes each time #5354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 155 additions & 30 deletions hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,14 @@ static abstract class Converter {
abstract int putLong(ByteBuffer buffer, int index, long val);
}

static abstract class CommonPrefixer {
abstract int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength);

abstract int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, ByteBuffer right,
int rightOffset, int rightLength);
}

static class ComparerHolder {
static final String UNSAFE_COMPARER_NAME = ComparerHolder.class.getName() + "$UnsafeComparer";

Expand Down Expand Up @@ -322,6 +330,111 @@ int putLong(ByteBuffer buffer, int index, long val) {
}
}

static class CommonPrefixerHolder {
static final String UNSAFE_COMMON_PREFIXER_NAME =
CommonPrefixerHolder.class.getName() + "$UnsafeCommonPrefixer";

static final CommonPrefixer BEST_COMMON_PREFIXER = getBestCommonPrefixer();

static CommonPrefixer getBestCommonPrefixer() {
try {
Class<? extends CommonPrefixer> theClass =
Class.forName(UNSAFE_COMMON_PREFIXER_NAME).asSubclass(CommonPrefixer.class);

return theClass.getConstructor().newInstance();
} catch (Throwable t) { // ensure we really catch *everything*
return PureJavaCommonPrefixer.INSTANCE;
}
}

static final class PureJavaCommonPrefixer extends CommonPrefixer {
static final PureJavaCommonPrefixer INSTANCE = new PureJavaCommonPrefixer();

private PureJavaCommonPrefixer() {
}

@Override
public int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength) {
int length = Math.min(leftLength, rightLength);
int result = 0;

while (
result < length
&& ByteBufferUtils.toByte(left, leftOffset + result) == right[rightOffset + result]
) {
result++;
}

return result;
}

@Override
int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, ByteBuffer right,
int rightOffset, int rightLength) {
int length = Math.min(leftLength, rightLength);
int result = 0;

while (
result < length && ByteBufferUtils.toByte(left, leftOffset + result)
== ByteBufferUtils.toByte(right, rightOffset + result)
) {
result++;
}

return result;
}
}

static final class UnsafeCommonPrefixer extends CommonPrefixer {

static {
if (!UNSAFE_UNALIGNED) {
throw new Error();
}
}

public UnsafeCommonPrefixer() {
}

@Override
public int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength) {
long offset1Adj;
Object refObj1 = null;
if (left.isDirect()) {
offset1Adj = leftOffset + UnsafeAccess.directBufferAddress(left);
} else {
offset1Adj = leftOffset + left.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj1 = left.array();
}
return findCommonPrefixUnsafe(refObj1, offset1Adj, leftLength, right,
rightOffset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET, rightLength);
}

@Override
public int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, ByteBuffer right,
int rightOffset, int rightLength) {
long offset1Adj, offset2Adj;
Object refObj1 = null, refObj2 = null;
if (left.isDirect()) {
offset1Adj = leftOffset + UnsafeAccess.directBufferAddress(left);
} else {
offset1Adj = leftOffset + left.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj1 = left.array();
}
if (right.isDirect()) {
offset2Adj = rightOffset + UnsafeAccess.directBufferAddress(right);
} else {
offset2Adj = rightOffset + right.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj2 = right.array();
}
return findCommonPrefixUnsafe(refObj1, offset1Adj, leftLength, refObj2, offset2Adj,
rightLength);
}
}
}

/**
* Similar to {@link WritableUtils#writeVLong(java.io.DataOutput, long)}, but writes to a
* {@link ByteBuffer}.
Expand Down Expand Up @@ -744,14 +857,7 @@ public static void copyFromBufferToBuffer(ByteBuffer in, ByteBuffer out, int sou
*/
public static int findCommonPrefix(byte[] left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength) {
int length = Math.min(leftLength, rightLength);
int result = 0;

while (result < length && left[leftOffset + result] == right[rightOffset + result]) {
result++;
}

return result;
return Bytes.findCommonPrefix(left, right, leftLength, rightLength, leftOffset, rightOffset);
}

/**
Expand All @@ -765,17 +871,8 @@ public static int findCommonPrefix(byte[] left, int leftOffset, int leftLength,
*/
public static int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength,
ByteBuffer right, int rightOffset, int rightLength) {
int length = Math.min(leftLength, rightLength);
int result = 0;

while (
result < length && ByteBufferUtils.toByte(left, leftOffset + result)
== ByteBufferUtils.toByte(right, rightOffset + result)
) {
result++;
}

return result;
return CommonPrefixerHolder.BEST_COMMON_PREFIXER.findCommonPrefix(left, leftOffset, leftLength,
right, rightOffset, rightLength);
}

/**
Expand All @@ -789,17 +886,8 @@ public static int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLeng
*/
public static int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength) {
int length = Math.min(leftLength, rightLength);
int result = 0;

while (
result < length
&& ByteBufferUtils.toByte(left, leftOffset + result) == right[rightOffset + result]
) {
result++;
}

return result;
return CommonPrefixerHolder.BEST_COMMON_PREFIXER.findCommonPrefix(left, leftOffset, leftLength,
right, rightOffset, rightLength);
}

/**
Expand Down Expand Up @@ -972,6 +1060,43 @@ static int compareToUnsafe(Object obj1, long o1, int l1, Object obj2, long o2, i
return l1 - l2;
}

static int findCommonPrefixUnsafe(Object left, long leftOffset, int leftLength, Object right,
long rightOffset, int rightLength) {
final int stride = 8;
final int minLength = Math.min(leftLength, rightLength);
int strideLimit = minLength & ~(stride - 1);
int result = 0;
int i;

for (i = 0; i < strideLimit; i += stride) {
long lw = HBasePlatformDependent.getLong(left, leftOffset + (long) i);
long rw = HBasePlatformDependent.getLong(right, rightOffset + (long) i);

if (lw != rw) {
if (!UnsafeAccess.LITTLE_ENDIAN) {
return result + (Long.numberOfLeadingZeros(lw ^ rw) / Bytes.SIZEOF_LONG);
} else {
return result + (Long.numberOfTrailingZeros(lw ^ rw) / Bytes.SIZEOF_LONG);
}
} else {
result += Bytes.SIZEOF_LONG;
}
}

// The epilogue to cover the last (minLength % stride) elements.
for (; i < minLength; i++) {
byte il = HBasePlatformDependent.getByte(left, leftOffset + i);
byte ir = HBasePlatformDependent.getByte(right, rightOffset + i);
if (il != ir) {
return result;
} else {
result++;
}
}

return result;
}

/**
* Reads a short value at the given buffer's offset.
* @param buffer input byte buffer to read
Expand Down
107 changes: 100 additions & 7 deletions hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1179,6 +1179,11 @@ static abstract class Converter {

}

static abstract class CommonPrefixer {
abstract int findCommonPrefix(byte[] left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength);
}

static Comparer<byte[]> lexicographicalComparerJavaImpl() {
return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
}
Expand Down Expand Up @@ -1453,6 +1458,99 @@ public int compareTo(byte[] buffer1, int offset1, int length1, byte[] buffer2, i
}
}

static class CommonPrefixerHolder {
static final String UNSAFE_COMMON_PREFIXER_NAME =
CommonPrefixerHolder.class.getName() + "$UnsafeCommonPrefixer";

static final CommonPrefixer BEST_COMMON_PREFIXER = getBestCommonPrefixer();

static CommonPrefixer getBestCommonPrefixer() {
try {
Class<? extends CommonPrefixer> theClass =
Class.forName(UNSAFE_COMMON_PREFIXER_NAME).asSubclass(CommonPrefixer.class);

return theClass.getConstructor().newInstance();
} catch (Throwable t) { // ensure we really catch *everything*
return CommonPrefixerHolder.PureJavaCommonPrefixer.INSTANCE;
}
}

static final class PureJavaCommonPrefixer extends CommonPrefixer {
static final PureJavaCommonPrefixer INSTANCE = new PureJavaCommonPrefixer();

private PureJavaCommonPrefixer() {
}

@Override
public int findCommonPrefix(byte[] left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength) {
int length = Math.min(leftLength, rightLength);
int result = 0;

while (result < length && left[leftOffset + result] == right[rightOffset + result]) {
result++;
}
return result;
}
}

static final class UnsafeCommonPrefixer extends CommonPrefixer {

static {
if (!UNSAFE_UNALIGNED) {
throw new Error();
}

// sanity check - this should never fail
if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) {
throw new AssertionError();
}
}

public UnsafeCommonPrefixer() {
}

@Override
public int findCommonPrefix(byte[] left, int leftOffset, int leftLength, byte[] right,
int rightOffset, int rightLength) {
final int stride = 8;
final int minLength = Math.min(leftLength, rightLength);
int strideLimit = minLength & ~(stride - 1);
final long leftOffsetAdj = leftOffset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
final long rightOffsetAdj = rightOffset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
int result = 0;
int i;

for (i = 0; i < strideLimit; i += stride) {
long lw = HBasePlatformDependent.getLong(left, leftOffsetAdj + i);
long rw = HBasePlatformDependent.getLong(right, rightOffsetAdj + i);
if (lw != rw) {
if (!UnsafeAccess.LITTLE_ENDIAN) {
return result + (Long.numberOfLeadingZeros(lw ^ rw) / Bytes.SIZEOF_LONG);
} else {
return result + (Long.numberOfTrailingZeros(lw ^ rw) / Bytes.SIZEOF_LONG);
}
} else {
result += Bytes.SIZEOF_LONG;
}
}

// The epilogue to cover the last (minLength % stride) elements.
for (; i < minLength; i++) {
int il = (left[leftOffset + i]);
int ir = (right[rightOffset + i]);
if (il != ir) {
return result;
} else {
result++;
}
}

return result;
}
}
}

/**
* Lexicographically determine the equality of two arrays.
* @param left left operand
Expand Down Expand Up @@ -2429,12 +2527,7 @@ public static int searchDelimiterIndexInReverse(final byte[] b, final int offset

public static int findCommonPrefix(byte[] left, byte[] right, int leftLength, int rightLength,
int leftOffset, int rightOffset) {
int length = Math.min(leftLength, rightLength);
int result = 0;

while (result < length && left[leftOffset + result] == right[rightOffset + result]) {
result++;
}
return result;
return CommonPrefixerHolder.BEST_COMMON_PREFIXER.findCommonPrefix(left, leftOffset, leftLength,
right, rightOffset, rightLength);
}
}
31 changes: 31 additions & 0 deletions hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,37 @@ public void testEquals() {
assertTrue(ByteBufferUtils.equals(bb, 0, a.length, a, 0, a.length));
}

@Test
public void testFindCommonPrefix() {
ByteBuffer bb1 = ByteBuffer.allocate(135);
ByteBuffer bb2 = ByteBuffer.allocate(135);
ByteBuffer bb3 = ByteBuffer.allocateDirect(135);
byte[] b = new byte[71];

fillBB(bb1, (byte) 5);
fillBB(bb2, (byte) 5);
fillBB(bb3, (byte) 5);
fillArray(b, (byte) 5);

assertEquals(135,
ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()));
assertEquals(71, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length));
assertEquals(135,
ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb3, 0, bb3.remaining()));
assertEquals(71, ByteBufferUtils.findCommonPrefix(bb3, 0, bb3.remaining(), b, 0, b.length));

b[13] = 9;
assertEquals(13, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length));

bb2.put(134, (byte) 6);
assertEquals(134,
ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()));

bb2.put(6, (byte) 4);
assertEquals(6,
ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()));
}

private static void fillBB(ByteBuffer bb, byte b) {
for (int i = bb.position(); i < bb.limit(); i++) {
bb.put(i, b);
Expand Down
Loading