From 3c71496e8766005bbb22892b3679d710f89c4a24 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sat, 19 Oct 2019 23:43:02 +0530 Subject: [PATCH 1/4] HBASE-22679 : Revamping CellUtil --- .../hadoop/hbase/protobuf/ProtobufUtil.java | 8 +- .../hadoop/hbase/ByteBufferKeyValue.java | 2 +- .../org/apache/hadoop/hbase/CellUtil.java | 728 +----------------- .../apache/hadoop/hbase/PrivateCellUtil.java | 31 +- .../io/encoding/BufferedDataBlockEncoder.java | 2 +- .../org/apache/hadoop/hbase/TestCellUtil.java | 32 +- .../hbase/util/MapReduceExtendedCell.java | 2 +- .../hadoop/hbase/mapreduce/TestSyncTable.java | 4 +- .../hadoop/hbase/util/CompressionTest.java | 14 +- .../hbase/HFilePerformanceEvaluation.java | 18 +- .../hadoop/hbase/TestTagRewriteCell.java | 9 +- .../client/TestAppendFromClientSide.java | 12 +- .../client/TestIncrementsFromClientSide.java | 12 +- .../client/TestResultFromCoprocessor.java | 11 +- .../hadoop/hbase/io/hfile/TestHFile.java | 156 +++- .../hbase/io/hfile/TestHFileBlockIndex.java | 19 +- .../TestHFileInlineToRootChunkConversion.java | 21 +- .../hbase/regionserver/TestBulkLoad.java | 16 +- .../hbase/regionserver/TestHRegion.java | 21 +- .../regionserver/TestHRegionReplayEvents.java | 12 +- .../hbase/regionserver/TestStoreScanner.java | 111 ++- .../regionserver/wal/TestFSWALEntry.java | 48 +- .../hadoop/hbase/thrift2/ThriftUtilities.java | 4 +- 23 files changed, 456 insertions(+), 837 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index ac0695b35ddd..a73a2bc2e302 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -687,8 +687,12 @@ private static T toDelta(Function supplier, Consu if (qv.hasTags()) { tags = qv.getTags().toByteArray(); } - consumer.accept(mutation, CellUtil.createCell(mutation.getRow(), family, qualifier, qv.getTimestamp(), - KeyValue.Type.Put, value, tags)); + consumer.accept(mutation, ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(mutation.getRow()).setFamily(family) + .setQualifier(qualifier).setTimestamp(qv.getTimestamp()) + .setType(KeyValue.Type.Put.getCode()).setValue(value) + .setTags(tags).setSequenceId(0) + .build()); } } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java index 8c6a2d588950..caf11c3253f5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java @@ -180,7 +180,7 @@ public int getValueLength() { @Override public byte[] getTagsArray() { - return CellUtil.cloneTags(this); + return PrivateCellUtil.cloneTags(this); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index fcf99aa19ad4..7ff920ec28c2 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -21,29 +21,19 @@ import static org.apache.hadoop.hbase.KeyValue.COLUMN_FAMILY_DELIMITER; import static org.apache.hadoop.hbase.KeyValue.COLUMN_FAMILY_DELIM_ARRAY; import static org.apache.hadoop.hbase.KeyValue.getDelimiter; -import static org.apache.hadoop.hbase.Tag.TAG_LENGTH_SIZE; -import java.io.DataOutput; -import java.io.DataOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; -import java.util.Optional; import org.apache.hadoop.hbase.KeyValue.Type; -import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.ByteBufferUtils; -import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.yetus.audience.InterfaceAudience.Private; - -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Utility methods helpful for slinging {@link Cell} instances. Some methods below are for internal @@ -59,48 +49,6 @@ public final class CellUtil { private CellUtil() { } - /******************* ByteRange *******************************/ - - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - */ - @Deprecated - public static ByteRange fillRowRange(Cell cell, ByteRange range) { - return PrivateCellUtil.fillRowRange(cell, range); - } - - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - */ - @Deprecated - public static ByteRange fillFamilyRange(Cell cell, ByteRange range) { - return PrivateCellUtil.fillFamilyRange(cell, range); - } - - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - */ - @Deprecated - public static ByteRange fillQualifierRange(Cell cell, ByteRange range) { - return PrivateCellUtil.fillQualifierRange(cell, range); - } - - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - */ - @Deprecated - public static ByteRange fillValueRange(Cell cell, ByteRange range) { - return PrivateCellUtil.fillValueRange(cell, range); - } - - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - */ - @Deprecated - public static ByteRange fillTagRange(Cell cell, ByteRange range) { - return PrivateCellUtil.fillTagRange(cell, range); - } - /***************** get individual arrays for tests ************/ public static byte[] cloneRow(Cell cell) { @@ -127,28 +75,6 @@ public static byte[] cloneValue(Cell cell) { return output; } - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - * Use {@link RawCell#cloneTags()} - */ - @Deprecated - public static byte[] cloneTags(Cell cell) { - return PrivateCellUtil.cloneTags(cell); - } - - /** - * Returns tag value in a new byte array. If server-side, use {@link Tag#getValueArray()} with - * appropriate {@link Tag#getValueOffset()} and {@link Tag#getValueLength()} instead to save on - * allocations. - * @param cell - * @return tag value in a new byte array. - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 - */ - @Deprecated - public static byte[] getTagArray(Cell cell) { - return PrivateCellUtil.cloneTags(cell); - } - /** * Makes a column in family:qualifier form from separate byte arrays. *

@@ -371,228 +297,6 @@ public static int copyValueTo(Cell cell, ByteBuffer destination, int destination return destinationOffset + vlen; } - /** - * Copies the tags info into the tag portion of the cell - * @param cell - * @param destination - * @param destinationOffset - * @return position after tags - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - */ - @Deprecated - public static int copyTagTo(Cell cell, byte[] destination, int destinationOffset) { - return PrivateCellUtil.copyTagsTo(cell, destination, destinationOffset); - } - - /** - * Copies the tags info into the tag portion of the cell - * @param cell - * @param destination - * @param destinationOffset - * @return position after tags - * @deprecated As of HBase-2.0. Will be removed in 3.0. - */ - @Deprecated - public static int copyTagTo(Cell cell, ByteBuffer destination, int destinationOffset) { - return PrivateCellUtil.copyTagsTo(cell, destination, destinationOffset); - } - - /********************* misc *************************************/ - - @Private - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0. - */ - @Deprecated - public static byte getRowByte(Cell cell, int index) { - return PrivateCellUtil.getRowByte(cell, index); - } - - /** - * @deprecated As of HBase-2.0. Will be removed in 3.0. - */ - @Deprecated - public static ByteBuffer getValueBufferShallowCopy(Cell cell) { - return PrivateCellUtil.getValueBufferShallowCopy(cell); - } - - /** - * @param cell - * @return cell's qualifier wrapped into a ByteBuffer. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static ByteBuffer getQualifierBufferShallowCopy(Cell cell) { - // No usage of this in code. - ByteBuffer buffer = ByteBuffer.wrap(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength()); - return buffer; - } - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use {@link CellBuilder} - * instead - */ - @Deprecated - public static Cell createCell(final byte[] row, final byte[] family, final byte[] qualifier, - final long timestamp, final byte type, final byte[] value) { - return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(row) - .setFamily(family) - .setQualifier(qualifier) - .setTimestamp(timestamp) - .setType(type) - .setValue(value) - .build(); - } - - /** - * Creates a cell with deep copy of all passed bytes. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use {@link CellBuilder} - * instead - */ - @Deprecated - public static Cell createCell(final byte[] rowArray, final int rowOffset, final int rowLength, - final byte[] familyArray, final int familyOffset, final int familyLength, - final byte[] qualifierArray, final int qualifierOffset, final int qualifierLength) { - // See createCell(final byte [] row, final byte [] value) for why we default Maximum type. - return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(rowArray, rowOffset, rowLength) - .setFamily(familyArray, familyOffset, familyLength) - .setQualifier(qualifierArray, qualifierOffset, qualifierLength) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY, 0, HConstants.EMPTY_BYTE_ARRAY.length) - .build(); - } - - /** - * Marked as audience Private as of 1.2.0. - * Creating a Cell with a memstoreTS/mvcc is an internal - * implementation detail not for public use. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use - * {@link ExtendedCellBuilder} instead - */ - @InterfaceAudience.Private - @Deprecated - public static Cell createCell(final byte[] row, final byte[] family, final byte[] qualifier, - final long timestamp, final byte type, final byte[] value, final long memstoreTS) { - return createCell(row, family, qualifier, timestamp, type, value, null, memstoreTS); - } - - /** - * Marked as audience Private as of 1.2.0. - * Creating a Cell with tags and a memstoreTS/mvcc is an - * internal implementation detail not for public use. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use - * {@link ExtendedCellBuilder} instead - */ - @InterfaceAudience.Private - @Deprecated - public static Cell createCell(final byte[] row, final byte[] family, final byte[] qualifier, - final long timestamp, final byte type, final byte[] value, byte[] tags, - final long memstoreTS) { - return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(row) - .setFamily(family) - .setQualifier(qualifier) - .setTimestamp(timestamp) - .setType(type) - .setValue(value) - .setTags(tags) - .setSequenceId(memstoreTS) - .build(); - } - - /** - * Marked as audience Private as of 1.2.0. - * Creating a Cell with tags is an internal implementation detail not for public use. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use - * {@link ExtendedCellBuilder} instead - */ - @InterfaceAudience.Private - @Deprecated - public static Cell createCell(final byte[] row, final byte[] family, final byte[] qualifier, - final long timestamp, Type type, final byte[] value, byte[] tags) { - return createCell(row, family, qualifier, timestamp, type.getCode(), value, tags, 0); - } - - /** - * Create a Cell with specific row. Other fields defaulted. - * @param row - * @return Cell with passed row but all other fields are arbitrary - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use {@link CellBuilder} - * instead - */ - @Deprecated - public static Cell createCell(final byte[] row) { - return createCell(row, HConstants.EMPTY_BYTE_ARRAY); - } - - /** - * Create a Cell with specific row and value. Other fields are defaulted. - * @param row - * @param value - * @return Cell with passed row and value but all other fields are arbitrary - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use {@link CellBuilder} - * instead - */ - @Deprecated - public static Cell createCell(final byte[] row, final byte[] value) { - // An empty family + empty qualifier + Type.Minimum is used as flag to indicate last on row. - // See the CellComparator and KeyValue comparator. Search for compareWithoutRow. - // Lets not make a last-on-row key as default but at same time, if you are making a key - // without specifying type, etc., flag it as weird by setting type to be Maximum. - return createCell(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, - HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum.getCode(), value); - } - - /** - * Create a Cell with specific row. Other fields defaulted. - * @param row - * @param family - * @param qualifier - * @return Cell with passed row but all other fields are arbitrary - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - * Use {@link CellBuilder} instead - */ - @Deprecated - public static Cell createCell(final byte [] row, final byte [] family, final byte [] qualifier) { - // See above in createCell(final byte [] row, final byte [] value) why we set type to Maximum. - return createCell(row, family, qualifier, - HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum.getCode(), HConstants.EMPTY_BYTE_ARRAY); - } - - /** - * Note : Now only CPs can create cell with tags using the CP environment - * @return A new cell which is having the extra tags also added to it. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - * - */ - @Deprecated - public static Cell createCell(Cell cell, List tags) { - return PrivateCellUtil.createCell(cell, tags); - } - - /** - * Now only CPs can create cell with tags using the CP environment - * @return A new cell which is having the extra tags also added to it. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static Cell createCell(Cell cell, byte[] tags) { - return PrivateCellUtil.createCell(cell, tags); - } - - /** - * Now only CPs can create cell with tags using the CP environment - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static Cell createCell(Cell cell, byte[] value, byte[] tags) { - return PrivateCellUtil.createCell(cell, value, tags); - } - /** * @param cellScannerables * @return CellScanner interface over cellIterables @@ -714,27 +418,6 @@ public boolean advance() { }; } - /** - * @param left - * @param right - * @return True if the rows in left and right Cells match - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - * Instead use {@link #matchingRows(Cell, Cell)} - */ - @Deprecated - public static boolean matchingRow(final Cell left, final Cell right) { - return matchingRows(left, right); - } - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - * Instead use {@link #matchingRows(Cell, byte[])} - */ - @Deprecated - public static boolean matchingRow(final Cell left, final byte[] buf) { - return matchingRows(left, buf); - } - public static boolean matchingRows(final Cell left, final byte[] buf) { if (buf == null) { return left.getRowLength() == 0; @@ -742,11 +425,6 @@ public static boolean matchingRows(final Cell left, final byte[] buf) { return PrivateCellUtil.matchingRows(left, buf, 0, buf.length); } - public static boolean matchingRow(final Cell left, final byte[] buf, final int offset, - final int length) { - return PrivateCellUtil.matchingRows(left, buf, offset, length); - } - public static boolean matchingFamily(final Cell left, final Cell right) { byte lfamlength = left.getFamilyLength(); byte rfamlength = right.getFamilyLength(); @@ -774,16 +452,7 @@ public static boolean matchingFamily(final Cell left, final byte[] buf) { if (buf == null) { return left.getFamilyLength() == 0; } - return matchingFamily(left, buf, 0, buf.length); - } - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean matchingFamily(final Cell left, final byte[] buf, final int offset, - final int length) { - return PrivateCellUtil.matchingFamily(left, buf, offset, length); + return PrivateCellUtil.matchingFamily(left, buf, 0, buf.length); } public static boolean matchingQualifier(final Cell left, final Cell right) { @@ -821,23 +490,7 @@ public static boolean matchingQualifier(final Cell left, final byte[] buf) { if (buf == null) { return left.getQualifierLength() == 0; } - return matchingQualifier(left, buf, 0, buf.length); - } - - /** - * Finds if the qualifier part of the cell and the KV serialized - * byte[] are equal - * @param left - * @param buf the serialized keyvalue format byte[] - * @param offset the offset of the qualifier in the byte[] - * @param length the length of the qualifier in the byte[] - * @return true if the qualifier matches, false otherwise - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean matchingQualifier(final Cell left, final byte[] buf, final int offset, - final int length) { - return PrivateCellUtil.matchingQualifier(left, buf, offset, length); + return PrivateCellUtil.matchingQualifier(left, buf, 0, buf.length); } public static boolean matchingColumn(final Cell left, final byte[] fam, final byte[] qual) { @@ -846,15 +499,6 @@ public static boolean matchingColumn(final Cell left, final byte[] fam, final by return matchingQualifier(left, qual); } - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean matchingColumn(final Cell left, final byte[] fam, final int foffset, - final int flength, final byte[] qual, final int qoffset, final int qlength) { - return PrivateCellUtil.matchingColumn(left, fam, foffset, flength, qual, qoffset, qlength); - } - public static boolean matchingColumn(final Cell left, final Cell right) { if (!matchingFamily(left, right)) return false; @@ -907,70 +551,6 @@ public static boolean isDelete(final Cell cell) { return PrivateCellUtil.isDelete(cell.getTypeByte()); } - /** - * @return True if a delete type, a {@link KeyValue.Type#Delete} or a - * {KeyValue.Type#DeleteFamily} or a - * {@link KeyValue.Type#DeleteColumn} KeyValue type. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean isDelete(final byte type) { - return Type.Delete.getCode() <= type - && type <= Type.DeleteFamily.getCode(); - } - - /** - * @return True if this cell is a {@link KeyValue.Type#Delete} type. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean isDeleteType(Cell cell) { - return cell.getTypeByte() == Type.Delete.getCode(); - } - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean isDeleteFamily(final Cell cell) { - return cell.getTypeByte() == Type.DeleteFamily.getCode(); - } - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean isDeleteFamilyVersion(final Cell cell) { - return cell.getTypeByte() == Type.DeleteFamilyVersion.getCode(); - } - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean isDeleteColumns(final Cell cell) { - return cell.getTypeByte() == Type.DeleteColumn.getCode(); - } - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean isDeleteColumnVersion(final Cell cell) { - return cell.getTypeByte() == Type.Delete.getCode(); - } - - /** - * - * @return True if this cell is a delete family or column type. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean isDeleteColumnOrFamily(Cell cell) { - int t = cell.getTypeByte(); - return t == Type.DeleteColumn.getCode() || t == Type.DeleteFamily.getCode(); - } - /** * @return True if this cell is a Put. */ @@ -979,141 +559,6 @@ public static boolean isPut(Cell cell) { return cell.getTypeByte() == Type.Put.getCode(); } - /** - * Estimate based on keyvalue's serialization format in the RPC layer. Note that there is an extra - * SIZEOF_INT added to the size here that indicates the actual length of the cell for cases where - * cell's are serialized in a contiguous format (For eg in RPCs). - * @param cell - * @return Estimate of the cell size in bytes plus an extra SIZEOF_INT indicating the - * actual cell length. - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static int estimatedSerializedSizeOf(final Cell cell) { - return PrivateCellUtil.estimatedSerializedSizeOf(cell); - } - - /** - * Calculates the serialized key size. We always serialize in the KeyValue's serialization - * format. - * @param cell the cell for which the key size has to be calculated. - * @return the key size - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static int estimatedSerializedSizeOfKey(final Cell cell) { - return PrivateCellUtil.estimatedSerializedSizeOfKey(cell); - } - - /** - * This is an estimate of the heap space occupied by a cell. When the cell is of type - * {@link HeapSize} we call {@link HeapSize#heapSize()} so cell can give a correct value. In other - * cases we just consider the bytes occupied by the cell components ie. row, CF, qualifier, - * timestamp, type, value and tags. - * @param cell - * @return estimate of the heap space - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static long estimatedHeapSizeOf(final Cell cell) { - return cell.heapSize(); - } - - /********************* tags *************************************/ - /** - * Util method to iterate through the tags - * - * @param tags - * @param offset - * @param length - * @return iterator for the tags - * @deprecated As of 2.0.0 and will be removed in 3.0.0 - * Instead use {@link PrivateCellUtil#tagsIterator(Cell)} - */ - @Deprecated - public static Iterator tagsIterator(final byte[] tags, final int offset, final int length) { - return new Iterator() { - private int pos = offset; - private int endOffset = offset + length - 1; - - @Override - public boolean hasNext() { - return this.pos < endOffset; - } - - @Override - public Tag next() { - if (hasNext()) { - int curTagLen = Bytes.readAsInt(tags, this.pos, Tag.TAG_LENGTH_SIZE); - Tag tag = new ArrayBackedTag(tags, pos, curTagLen + TAG_LENGTH_SIZE); - this.pos += Bytes.SIZEOF_SHORT + curTagLen; - return tag; - } - return null; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - - /** - * @param cell The Cell - * @return Tags in the given Cell as a List - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - * Use {@link RawCell#getTags()} - */ - @Deprecated - public static List getTags(Cell cell) { - return PrivateCellUtil.getTags(cell); - } - - /** - * Retrieve Cell's first tag, matching the passed in type - * - * @param cell The Cell - * @param type Type of the Tag to retrieve - * @return null if there is no tag of the passed in tag type - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - * Use {@link RawCell#getTag(byte)} - */ - @Deprecated - public static Tag getTag(Cell cell, byte type) { - Optional tag = PrivateCellUtil.getTag(cell, type); - if (tag.isPresent()) { - return tag.get(); - } else { - return null; - } - } - - /** - * Returns true if the first range start1...end1 overlaps with the second range - * start2...end2, assuming the byte arrays represent row keys - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean overlappingKeys(final byte[] start1, final byte[] end1, - final byte[] start2, final byte[] end2) { - return PrivateCellUtil.overlappingKeys(start1, end1, start2, end2); - } - - /** - * Sets the given seqId to the cell. - * Marked as audience Private as of 1.2.0. - * Setting a Cell sequenceid is an internal implementation detail not for general public use. - * @param cell - * @param seqId - * @throws IOException when the passed cell is not of type {@link ExtendedCell} - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static void setSequenceId(Cell cell, long seqId) throws IOException { - PrivateCellUtil.setSequenceId(cell, seqId); - } - /** * Sets the given timestamp to the cell. * @@ -1141,80 +586,6 @@ public static void setTimestamp(Cell cell, byte[] ts, int tsOffset) throws IOExc PrivateCellUtil.setTimestamp(cell, Bytes.toLong(ts, tsOffset)); } - /** - * Sets the given timestamp to the cell iff current timestamp is - * {@link HConstants#LATEST_TIMESTAMP}. - * @param cell - * @param ts - * @return True if cell timestamp is modified. - * @throws IOException when the passed cell is not of type {@link ExtendedCell} - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean updateLatestStamp(Cell cell, long ts) throws IOException { - return PrivateCellUtil.updateLatestStamp(cell, ts); - } - - /** - * Sets the given timestamp to the cell iff current timestamp is - * {@link HConstants#LATEST_TIMESTAMP}. - * @param cell - * @param ts buffer containing the timestamp value - * @param tsOffset offset to the new timestamp - * @return True if cell timestamp is modified. - * @throws IOException when the passed cell is not of type {@link ExtendedCell} - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static boolean updateLatestStamp(Cell cell, byte[] ts, int tsOffset) throws IOException { - return PrivateCellUtil.updateLatestStamp(cell, Bytes.toLong(ts, tsOffset)); - } - - - /** - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static int writeFlatKey(Cell cell, OutputStream out) throws IOException { - return PrivateCellUtil.writeFlatKey(cell, out); - } - - /** - * Writes the row from the given cell to the output stream excluding the common prefix - * @param out The dataoutputstream to which the data has to be written - * @param cell The cell whose contents has to be written - * @param rlength the row length - * @throws IOException - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static void writeRowSkippingBytes(DataOutputStream out, Cell cell, short rlength, - int commonPrefix) throws IOException { - PrivateCellUtil.writeRowSkippingBytes(out, cell, rlength, commonPrefix); - } - - /** - * Writes the qualifier from the given cell to the output stream excluding the common prefix - * @param out The dataoutputstream to which the data has to be written - * @param cell The cell whose contents has to be written - * @param qlength the qualifier length - * @throws IOException - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static void writeQualifierSkippingBytes(DataOutputStream out, Cell cell, - int qlength, int commonPrefix) throws IOException { - if (cell instanceof ByteBufferExtendedCell) { - ByteBufferUtils.copyBufferToStream((DataOutput)out, - ((ByteBufferExtendedCell) cell).getQualifierByteBuffer(), - ((ByteBufferExtendedCell) cell).getQualifierPosition() + commonPrefix, - qlength - commonPrefix); - } else { - out.write(cell.getQualifierArray(), cell.getQualifierOffset() + commonPrefix, - qlength - commonPrefix); - } - } - /** * @param cell * @return The Key portion of the passed cell as a String. @@ -1243,53 +614,6 @@ public static String getCellKeyAsString(Cell cell) { return sb.toString(); } - /** - * This method exists just to encapsulate how we serialize keys. To be replaced by a factory - * that we query to figure what the Cell implementation is and then, what serialization engine - * to use and further, how to serialize the key for inclusion in hfile index. TODO. - * @param cell - * @return The key portion of the Cell serialized in the old-school KeyValue way or null if - * passed a null cell - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static byte [] getCellKeySerializedAsKeyValueKey(final Cell cell) { - return PrivateCellUtil.getCellKeySerializedAsKeyValueKey(cell); - } - - /** - * Write rowkey excluding the common part. - * @param cell - * @param rLen - * @param commonPrefix - * @param out - * @throws IOException - * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. - */ - @Deprecated - public static void writeRowKeyExcludingCommon(Cell cell, short rLen, int commonPrefix, - DataOutputStream out) throws IOException { - PrivateCellUtil.writeRowKeyExcludingCommon(cell, rLen, commonPrefix, out); - } - - /** - * Find length of common prefix in keys of the cells, considering key as byte[] if serialized in - * {@link KeyValue}. The key format is <2 bytes rk len><rk><1 byte cf - * len><cf><qualifier><8 bytes timestamp><1 byte type> - * @param c1 the cell - * @param c2 the cell - * @param bypassFamilyCheck when true assume the family bytes same in both cells. Pass it as true - * when dealing with Cells in same CF so as to avoid some checks - * @param withTsType when true check timestamp and type bytes also. - * @return length of common prefix - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 - */ - @Deprecated - public static int findCommonPrefixInFlatKey(Cell c1, Cell c2, boolean bypassFamilyCheck, - boolean withTsType) { - return PrivateCellUtil.findCommonPrefixInFlatKey(c1, c2, bypassFamilyCheck, withTsType); - } - /** Returns a string representation of the cell */ public static String toString(Cell cell, boolean verbose) { if (cell == null) { @@ -1322,17 +646,6 @@ public static String toString(Cell cell, boolean verbose) { return builder.toString(); } - /***************** special cases ****************************/ - - /** - * special case for Cell.equals - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 - */ - @Deprecated - public static boolean equalsIgnoreMvccVersion(Cell a, Cell b) { - return PrivateCellUtil.equalsIgnoreMvccVersion(a, b); - } - /**************** equals ****************************/ public static boolean equals(Cell a, Cell b) { @@ -1344,14 +657,6 @@ public static boolean matchingTimestamp(Cell a, Cell b) { return CellComparator.getInstance().compareTimestamps(a.getTimestamp(), b.getTimestamp()) == 0; } - /** - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 - */ - @Deprecated - public static boolean matchingType(Cell a, Cell b) { - return PrivateCellUtil.matchingType(a, b); - } - /** * Compares the row of two keyvalues for equality * @param left @@ -1449,35 +754,6 @@ public final static int compareQualifiers(Cell left, byte[] right, int rOffset, left.getQualifierLength(), right, rOffset, rLength); } - /** - * Used when a cell needs to be compared with a key byte[] such as cases of finding the index from - * the index block, bloom keys from the bloom blocks This byte[] is expected to be serialized in - * the KeyValue serialization format If the KeyValue (Cell's) serialization format changes this - * method cannot be used. - * @param comparator the cell comparator - * @param left the cell to be compared - * @param key the serialized key part of a KeyValue - * @param offset the offset in the key byte[] - * @param length the length of the key byte[] - * @return an int greater than 0 if left is greater than right lesser than 0 if left is lesser - * than right equal to 0 if left is equal to right - * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 - */ - @VisibleForTesting - @Deprecated - public static final int compare(CellComparator comparator, Cell left, byte[] key, int offset, - int length) { - // row - short rrowlength = Bytes.toShort(key, offset); - int c = comparator.compareRows(left, key, offset + Bytes.SIZEOF_SHORT, rrowlength); - if (c != 0) return c; - - // Compare the rest of the two KVs without making any assumptions about - // the common prefix. This function will not compare rows anyway, so we - // don't need to tell it that the common prefix includes the row. - return PrivateCellUtil.compareWithoutRow(comparator, left, key, offset, length, rrowlength); - } - /** * Compares the cell's family with the given byte[] * @param left the cell for which the family has to be compared diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java index 59c60730e43d..f85fbefedb87 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java @@ -958,7 +958,34 @@ public static Iterator tagsIterator(final Cell cell) { return tagsIterator(((ByteBufferExtendedCell) cell).getTagsByteBuffer(), ((ByteBufferExtendedCell) cell).getTagsPosition(), tagsLength); } - return CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); + + return new Iterator() { + private int offset = cell.getTagsOffset(); + private int pos = offset; + private int endOffset = offset + cell.getTagsLength() - 1; + + @Override + public boolean hasNext() { + return this.pos < endOffset; + } + + @Override + public Tag next() { + if (hasNext()) { + byte[] tags = cell.getTagsArray(); + int curTagLen = Bytes.readAsInt(tags, this.pos, Tag.TAG_LENGTH_SIZE); + Tag tag = new ArrayBackedTag(tags, pos, curTagLen + TAG_LENGTH_SIZE); + this.pos += Bytes.SIZEOF_SHORT + curTagLen; + return tag; + } + return null; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; } public static Iterator tagsIterator(final ByteBuffer tags, final int offset, @@ -1501,7 +1528,7 @@ public int getValueLength() { @Override public byte[] getTagsArray() { - return CellUtil.cloneTags(this); + return PrivateCellUtil.cloneTags(this); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index 7fe2b7129668..bb22ad83891b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -598,7 +598,7 @@ public int getValueLength() { @Override public byte[] getTagsArray() { - return CellUtil.cloneTags(this); + return PrivateCellUtil.cloneTags(this); } @Override diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java index 6f477b848e9b..0ec62c1876f2 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java @@ -382,19 +382,30 @@ public void testToString() { // Make a KeyValue and a Cell and see if same toString result. KeyValue kv = new KeyValue(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY); - Cell cell = CellUtil.createCell(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, - ts, KeyValue.Type.Minimum.getCode(), HConstants.EMPTY_BYTE_ARRAY); + Cell cell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(row) + .setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(ts) + .setType(KeyValue.Type.Minimum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build(); String cellToString = CellUtil.getCellKeyAsString(cell); assertEquals(kv.toString(), cellToString); // Do another w/ non-null family. byte [] f = new byte [] {'f'}; byte [] q = new byte [] {'q'}; kv = new KeyValue(row, f, q, ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY); - cell = CellUtil.createCell(row, f, q, ts, KeyValue.Type.Minimum.getCode(), - HConstants.EMPTY_BYTE_ARRAY); + cell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(row) + .setFamily(f) + .setQualifier(q) + .setTimestamp(ts) + .setType(KeyValue.Type.Minimum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build(); cellToString = CellUtil.getCellKeyAsString(cell); assertEquals(kv.toString(), cellToString); - } @Test @@ -407,8 +418,15 @@ public void testToString1() { String value = "test.value"; long seqId = 1042; - Cell cell = CellUtil.createCell(Bytes.toBytes(row), Bytes.toBytes(family), - Bytes.toBytes(qualifier), timestamp, type.getCode(), Bytes.toBytes(value), seqId); + Cell cell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes(row)) + .setFamily(Bytes.toBytes(family)) + .setQualifier(Bytes.toBytes(qualifier)) + .setTimestamp(timestamp) + .setType(type.getCode()) + .setValue(Bytes.toBytes(value)) + .setSequenceId(seqId) + .build(); String nonVerbose = CellUtil.toString(cell, false); String verbose = CellUtil.toString(cell, true); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceExtendedCell.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceExtendedCell.java index 175c0042fa37..ca0d5f63a078 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceExtendedCell.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceExtendedCell.java @@ -206,7 +206,7 @@ public ByteBuffer getTagsByteBuffer() { if (cell instanceof ByteBufferExtendedCell) { return ((ByteBufferExtendedCell) this.cell).getTagsByteBuffer(); } else { - return ByteBuffer.wrap(CellUtil.cloneTags(this.cell)); + return ByteBuffer.wrap(PrivateCellUtil.cloneTags(this.cell)); } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java index ad02039fb920..8e6344535d6b 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java @@ -283,7 +283,7 @@ private void assertTargetDoDeletesFalse(int expectedRows, TableName Cell sourceCell = sourceCells[j]; Cell targetCell = targetCells[j]; try { - if (!CellUtil.matchingRow(sourceCell, targetCell)) { + if (!CellUtil.matchingRows(sourceCell, targetCell)) { Assert.fail("Rows don't match"); } if (!CellUtil.matchingFamily(sourceCell, targetCell)) { @@ -384,7 +384,7 @@ private void assertTargetDoPutsFalse(int expectedRows, TableName Cell sourceCell = sourceCells[j]; Cell targetCell = targetCells[j]; try { - if (!CellUtil.matchingRow(sourceCell, targetCell)) { + if (!CellUtil.matchingRows(sourceCell, targetCell)) { Assert.fail("Rows don't match"); } if (!CellUtil.matchingFamily(sourceCell, targetCell)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java index fbaab3d32909..c21c80b11bd3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java @@ -22,7 +22,11 @@ import java.util.Locale; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; @@ -31,7 +35,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -128,7 +131,14 @@ public static void doSmokeTest(FileSystem fs, Path path, String codec) .create(); // Write any-old Cell... final byte [] rowKey = Bytes.toBytes("compressiontestkey"); - Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval")); + Cell c = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(rowKey) + .setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(Bytes.toBytes("compressiontestval")) + .build(); writer.append(c); writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval")); writer.close(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java index 3ee6f7d6d4c4..04ab8003c373 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java @@ -91,11 +91,25 @@ static Cell createCell(final int i, final byte [] value) { } static Cell createCell(final byte [] keyRow) { - return CellUtil.createCell(keyRow); + return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(keyRow) + .setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build(); } static Cell createCell(final byte [] keyRow, final byte [] value) { - return CellUtil.createCell(keyRow, value); + return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(keyRow) + .setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(value) + .build(); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestTagRewriteCell.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestTagRewriteCell.java index d852d2db2ca3..92f4a8dc1013 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestTagRewriteCell.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestTagRewriteCell.java @@ -35,7 +35,14 @@ public class TestTagRewriteCell { @Test public void testHeapSize() { - Cell originalCell = CellUtil.createCell(Bytes.toBytes("row"), Bytes.toBytes("value")); + Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("row")) + .setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(Bytes.toBytes("value")) + .build(); final int fakeTagArrayLength = 10; Cell trCell = PrivateCellUtil.createCell(originalCell, new byte[fakeTagArrayLength]); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAppendFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAppendFromClientSide.java index 186b51f1e1fe..b5680a6fffbd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAppendFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAppendFromClientSide.java @@ -22,7 +22,8 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; @@ -73,7 +74,14 @@ public void testAppendWithCustomTimestamp() throws IOException { Table table = TEST_UTIL.createTable(TABLENAME, FAMILY); long timestamp = 999; Append append = new Append(ROW); - append.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L))); + append.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(ROW) + .setFamily(FAMILY) + .setQualifier(QUALIFIER) + .setTimestamp(timestamp) + .setType(KeyValue.Type.Put.getCode()) + .setValue(Bytes.toBytes(100L)) + .build()); Result r = table.append(append); assertEquals(1, r.size()); assertEquals(timestamp, r.rawCells()[0].getTimestamp()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIncrementsFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIncrementsFromClientSide.java index 4b2886ed6e6f..8395aa2c04f1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIncrementsFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIncrementsFromClientSide.java @@ -30,8 +30,9 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -465,7 +466,14 @@ public void testIncrementWithCustomTimestamp() throws IOException { Table table = TEST_UTIL.createTable(TABLENAME, FAMILY); long timestamp = 999; Increment increment = new Increment(ROW); - increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L))); + increment.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(ROW) + .setFamily(FAMILY) + .setQualifier(QUALIFIER) + .setTimestamp(timestamp) + .setType(KeyValue.Type.Put.getCode()) + .setValue(Bytes.toBytes(100L)) + .build()); Result r = table.increment(increment); assertEquals(1, r.size()); assertEquals(timestamp, r.rawCells()[0].getTimestamp()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResultFromCoprocessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResultFromCoprocessor.java index 27522f56a9e0..bbc27210826d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResultFromCoprocessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResultFromCoprocessor.java @@ -23,8 +23,10 @@ import java.util.Arrays; import java.util.Optional; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; @@ -55,8 +57,13 @@ public class TestResultFromCoprocessor { private static final byte[] QUAL = Bytes.toBytes("qual"); private static final byte[] VALUE = Bytes.toBytes(100L); private static final byte[] FIXED_VALUE = Bytes.toBytes("fixed_value"); - private static final Cell FIXED_CELL = CellUtil.createCell(ROW, FAMILY, - QUAL, 0, KeyValue.Type.Put.getCode(), FIXED_VALUE); + private static final Cell FIXED_CELL = ExtendedCellBuilderFactory + .create(CellBuilderType.DEEP_COPY) + .setRow(ROW).setFamily(FAMILY) + .setQualifier(QUAL).setTimestamp(0) + .setType(KeyValue.Type.Put.getCode()) + .setValue(FIXED_VALUE) + .build(); private static final Result FIXED_RESULT = Result.create(Arrays.asList(FIXED_CELL)); private static final TableName TABLE_NAME = TableName.valueOf("TestResultFromCoprocessor"); @BeforeClass diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 1b53da0df59e..54b0a467f83a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -47,8 +47,10 @@ import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -608,18 +610,22 @@ public void testCompressionOrdinance() { @Test public void testShortMidpointSameQual() { - Cell left = CellUtil.createCell(Bytes.toBytes("a"), - Bytes.toBytes("a"), - Bytes.toBytes("a"), - 11, - KeyValue.Type.Maximum.getCode(), - HConstants.EMPTY_BYTE_ARRAY); - Cell right = CellUtil.createCell(Bytes.toBytes("a"), - Bytes.toBytes("a"), - Bytes.toBytes("a"), - 9, - KeyValue.Type.Maximum.getCode(), - HConstants.EMPTY_BYTE_ARRAY); + Cell left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")) + .setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(11) + .setType(Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build(); + Cell right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")) + .setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(9) + .setType(Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build(); Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); @@ -627,53 +633,133 @@ public void testShortMidpointSameQual() { @Test public void testGetShortMidpoint() { - Cell left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - Cell right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + Cell left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + Cell right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("b")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); - left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("g")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("i")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("bbbbbbb")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); assertEquals(1, mid.getRowLength()); - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("b")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("aaaaaaaa")) + .setQualifier(Bytes.toBytes("b")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); assertEquals(2, mid.getFamilyLength()); - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("aaaaaaaaa")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); assertEquals(2, mid.getQualifierLength()); - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("b")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); @@ -681,8 +767,18 @@ public void testGetShortMidpoint() { // Assert that if meta comparator, it returns the right cell -- i.e. no // optimization done. - left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); + left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("g")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("i")).setFamily(Bytes.toBytes("a")) + .setQualifier(Bytes.toBytes("a")) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.META_COMPARATOR, left, right); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index faef3864c1b5..48c3aafaa3af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -37,11 +37,14 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -763,7 +766,13 @@ public void testIntermediateLevelIndicesWithLargeKeys(int minNumEntries) throws byte[] b = Bytes.toBytes(i); System.arraycopy(b, 0, rowkey, rowkey.length - b.length, b.length); keys.add(rowkey); - hfw.append(CellUtil.createCell(rowkey)); + hfw.append(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(rowkey).setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); } hfw.close(); @@ -771,7 +780,13 @@ public void testIntermediateLevelIndicesWithLargeKeys(int minNumEntries) throws // Scanner doesn't do Cells yet. Fix. HFileScanner scanner = reader.getScanner(true, true); for (int i = 0; i < keys.size(); ++i) { - scanner.seekTo(CellUtil.createCell(keys.get(i))); + scanner.seekTo(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(keys.get(i)).setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); } reader.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java index 8102f111e949..315f36fb17a9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java @@ -22,9 +22,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -80,7 +83,13 @@ public void testWriteHFile() throws Exception { byte[] k = Bytes.toBytes(keyStr); keys.add(k); byte[] v = Bytes.toBytes("value" + i); - hfw.append(CellUtil.createCell(k, v)); + hfw.append(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(k) + .setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(v).build()); } hfw.close(); @@ -88,7 +97,13 @@ public void testWriteHFile() throws Exception { // Scanner doesn't do Cells yet. Fix. HFileScanner scanner = reader.getScanner(true, true); for (int i = 0; i < keys.size(); ++i) { - scanner.seekTo(CellUtil.createCell(keys.get(i))); + scanner.seekTo(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(keys.get(i)) + .setFamily(HConstants.EMPTY_BYTE_ARRAY) + .setQualifier(HConstants.EMPTY_BYTE_ARRAY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY).build()); } reader.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java index a1a9373430f4..4893982f9373 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java @@ -40,8 +40,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -287,12 +289,14 @@ private String createHFileForFamilies(byte[] family) throws IOException { hFileFactory.withFileContext(new HFileContext()); HFile.Writer writer = hFileFactory.create(); try { - writer.append(new KeyValue(CellUtil.createCell(randomBytes, - family, - randomBytes, - 0L, - KeyValue.Type.Put.getCode(), - randomBytes))); + writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(randomBytes) + .setFamily(family) + .setQualifier(randomBytes) + .setTimestamp(0L) + .setType(KeyValue.Type.Put.getCode()) + .setValue(randomBytes) + .build())); } finally { writer.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 034c4b992d06..4e7c5ad66c41 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.DroppedSnapshotException; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -2366,12 +2367,24 @@ public void testDataInMemoryWithoutWAL() throws IOException { region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES); - Cell originalCell = CellUtil.createCell(row, COLUMN_FAMILY_BYTES, qual1, - System.currentTimeMillis(), KeyValue.Type.Put.getCode(), value1); + Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(row) + .setFamily(COLUMN_FAMILY_BYTES) + .setQualifier(qual1) + .setTimestamp(System.currentTimeMillis()) + .setType(KeyValue.Type.Put.getCode()) + .setValue(value1) + .build(); final long originalSize = originalCell.getSerializedSize(); - Cell addCell = CellUtil.createCell(row, COLUMN_FAMILY_BYTES, qual1, - System.currentTimeMillis(), KeyValue.Type.Put.getCode(), Bytes.toBytes("xxxxxxxxxx")); + Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(row) + .setFamily(COLUMN_FAMILY_BYTES) + .setQualifier(qual1) + .setTimestamp(System.currentTimeMillis()) + .setType(KeyValue.Type.Put.getCode()) + .setValue(Bytes.toBytes("xxxxxxxxxx")) + .build(); final long addSize = addCell.getSerializedSize(); LOG.info("originalSize:" + originalSize diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java index 92ecd861de42..b08877e4fa62 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java @@ -45,7 +45,9 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -1659,8 +1661,14 @@ private String createHFileForFamilies(Path testPath, byte[] family, hFileFactory.withFileContext(new HFileContext()); HFile.Writer writer = hFileFactory.create(); try { - writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0L, - KeyValue.Type.Put.getCode(), valueBytes))); + writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(valueBytes) + .setFamily(family) + .setQualifier(valueBytes) + .setTimestamp(0L) + .setType(KeyValue.Type.Put.getCode()) + .setValue(valueBytes) + .build())); } finally { writer.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index 84bdc0a2d21c..85acb1678798 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.apache.hadoop.hbase.CellUtil.createCell; import static org.apache.hadoop.hbase.KeyValueTestUtil.create; import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture; import static org.junit.Assert.assertEquals; @@ -36,8 +35,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -107,31 +108,73 @@ public class TestStoreScanner { * we do Gets, StoreScanner#optimize, and what we do on (faked) block boundaries. */ private static final Cell[] CELL_GRID = new Cell [] { - createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(ONE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(ONE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(ONE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(TWO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(THREE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), // Offset 4 CELL_GRID_BLOCK2_BOUNDARY - createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(TWO, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(TWO, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(TWO_POINT_TWO, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(TWO_POINT_TWO, CF, ZERO_POINT_ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(TWO_POINT_TWO, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) + .setFamily(CF).setQualifier(ONE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) + .setFamily(CF).setQualifier(TWO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) + .setFamily(CF).setQualifier(THREE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) + .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO) + .setFamily(CF).setQualifier(ZERO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO) + .setFamily(CF).setQualifier(ZERO_POINT_ZERO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO) + .setFamily(CF).setQualifier(FIVE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), // Offset 11! CELL_GRID_BLOCK3_BOUNDARY - createCell(THREE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(THREE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(THREE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(THREE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) + .setFamily(CF).setQualifier(ONE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) + .setFamily(CF).setQualifier(TWO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) + .setFamily(CF).setQualifier(THREE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) + .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), // Offset 15 CELL_GRID_BLOCK4_BOUNDARY - createCell(FOUR, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(FOUR, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(FOUR, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(FOUR, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) + .setFamily(CF).setQualifier(ONE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) + .setFamily(CF).setQualifier(TWO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) + .setFamily(CF).setQualifier(THREE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) + .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), // Offset 19 CELL_GRID_BLOCK5_BOUNDARY - createCell(FOUR, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(FIVE, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) + .setFamily(CF).setQualifier(FIVE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FIVE) + .setFamily(CF).setQualifier(ZERO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), }; private static class KeyValueHeapWithCount extends KeyValueHeap { @@ -218,13 +261,25 @@ public Cell getNextIndexedKey() { private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4; private static final Cell[] CELL_WITH_VERSIONS = new Cell [] { - createCell(ONE, CF, ONE, 2L, KeyValue.Type.Put.getCode(), VALUE), - createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(ONE, CF, TWO, 2L, KeyValue.Type.Put.getCode(), VALUE), - createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(ONE).setTimestamp(2L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(ONE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(TWO).setTimestamp(2L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) + .setFamily(CF).setQualifier(TWO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY - createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) + .setFamily(CF).setQualifier(ONE).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) + .setFamily(CF).setQualifier(TWO).setTimestamp(1L) + .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), }; private static class CellWithVersionsStoreScanner extends StoreScanner { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java index 993934d2384a..6c45bd4f2dc3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java @@ -22,8 +22,11 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -48,20 +51,51 @@ public void testCollectFamilies() { List cells = new ArrayList<>(); assertEquals(0, FSWALEntry.collectFamilies(cells).size()); - cells.add(CellUtil.createCell(family0, family0, family0)); + cells.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(family0).setFamily(family0).setQualifier(family0) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); assertEquals(1, FSWALEntry.collectFamilies(cells).size()); - cells.add(CellUtil.createCell(family1, family1, family1)); + cells.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(family1).setFamily(family1).setQualifier(family1) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); assertEquals(2, FSWALEntry.collectFamilies(cells).size()); - cells.add(CellUtil.createCell(family0, family0, family0)); - cells.add(CellUtil.createCell(family1, family1, family1)); + cells.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(family0).setFamily(family0).setQualifier(family0) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); + cells.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(family1).setFamily(family1).setQualifier(family1) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); assertEquals(2, FSWALEntry.collectFamilies(cells).size()); - cells.add(CellUtil.createCell(family2, family2, family2)); + cells.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(family2).setFamily(family2).setQualifier(family2) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); assertEquals(3, FSWALEntry.collectFamilies(cells).size()); - cells.add(CellUtil.createCell(WALEdit.METAFAMILY, WALEdit.METAFAMILY, WALEdit.METAFAMILY)); + cells.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(WALEdit.METAFAMILY).setFamily(WALEdit.METAFAMILY) + .setQualifier(WALEdit.METAFAMILY) + .setTimestamp(HConstants.LATEST_TIMESTAMP) + .setType(KeyValue.Type.Maximum.getCode()) + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build()); assertEquals(3, FSWALEntry.collectFamilies(cells).size()); } } diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java index d18a74b1353a..8371d9cebd00 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -1291,7 +1291,7 @@ public static TPut putFromHBase(Put in) { .setTimestamp(cell.getTimestamp()) .setValue(CellUtil.cloneValue(cell)); if (cell.getTagsLength() != 0) { - columnValue.setTags(CellUtil.cloneTags(cell)); + columnValue.setTags(PrivateCellUtil.cloneTags(cell)); } out.addToColumnValues(columnValue); } @@ -1356,7 +1356,7 @@ public static TAppend appendFromHBase(Append in) throws IOException { .setTimestamp(cell.getTimestamp()) .setValue(CellUtil.cloneValue(cell)); if (cell.getTagsLength() != 0) { - columnValue.setTags(CellUtil.cloneTags(cell)); + columnValue.setTags(PrivateCellUtil.cloneTags(cell)); } out.addToColumns(columnValue); } From 962ec7aef86fa866606222a8547dccf9369bdfac Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sun, 20 Oct 2019 13:00:51 +0530 Subject: [PATCH 2/4] checkstyle fix --- .../hadoop/hbase/io/hfile/TestHFile.java | 205 ++++++------------ 1 file changed, 65 insertions(+), 140 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 54b0a467f83a..0d76fe09d423 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -631,164 +631,89 @@ public void testShortMidpointSameQual() { assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); } - @Test - public void testGetShortMidpoint() { - Cell left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) + private Cell getCell(byte[] row, byte[] family, byte[] qualifier) { + return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(row) + .setFamily(family) + .setQualifier(qualifier) .setTimestamp(HConstants.LATEST_TIMESTAMP) .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - Cell right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); + .setValue(HConstants.EMPTY_BYTE_ARRAY) + .build(); + } - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("b")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + @Test + public void testGetShortMidpoint() { + Cell left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + Cell right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); + left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); - - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("g")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("i")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); + left = getCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); - - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("bbbbbbb")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); + left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); assertEquals(1, mid.getRowLength()); - - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("b")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); - - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("aaaaaaaa")) - .setQualifier(Bytes.toBytes("b")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); + left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); assertEquals(2, mid.getFamilyLength()); - - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("aaaaaaaaa")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); assertEquals(2, mid.getQualifierLength()); - - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("a")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("b")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); assertEquals(1, mid.getQualifierLength()); // Assert that if meta comparator, it returns the right cell -- i.e. no // optimization done. - left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("g")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); - right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) - .setRow(Bytes.toBytes("i")).setFamily(Bytes.toBytes("a")) - .setQualifier(Bytes.toBytes("a")) - .setTimestamp(HConstants.LATEST_TIMESTAMP) - .setType(KeyValue.Type.Maximum.getCode()) - .setValue(HConstants.EMPTY_BYTE_ARRAY).build(); + left = getCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); + right = getCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.META_COMPARATOR, left, right); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); - assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); - - /** - * See HBASE-7845 - */ - byte[] rowA = Bytes.toBytes("rowA"); - byte[] rowB = Bytes.toBytes("rowB"); - + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); + assertTrue(PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); byte[] family = Bytes.toBytes("family"); byte[] qualA = Bytes.toBytes("qfA"); byte[] qualB = Bytes.toBytes("qfB"); From 2ef30f3ca96a6234d9afd0dd89be8fcd23bb77d8 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 21 Oct 2019 14:19:28 +0530 Subject: [PATCH 3/4] incorporating review --- .../src/main/java/org/apache/hadoop/hbase/CellUtil.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index 7ff920ec28c2..fecd71a755d7 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -425,6 +425,11 @@ public static boolean matchingRows(final Cell left, final byte[] buf) { return PrivateCellUtil.matchingRows(left, buf, 0, buf.length); } + public static boolean matchingRow(final Cell left, final byte[] buf, final int offset, + final int length) { + return PrivateCellUtil.matchingRows(left, buf, offset, length); + } + public static boolean matchingFamily(final Cell left, final Cell right) { byte lfamlength = left.getFamilyLength(); byte rfamlength = right.getFamilyLength(); From 52d765fd0129263ca7ebcbfeac272f0c6c45857e Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 21 Oct 2019 14:21:37 +0530 Subject: [PATCH 4/4] minor indentation --- .../src/main/java/org/apache/hadoop/hbase/CellUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index fecd71a755d7..64d58611f413 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -426,7 +426,7 @@ public static boolean matchingRows(final Cell left, final byte[] buf) { } public static boolean matchingRow(final Cell left, final byte[] buf, final int offset, - final int length) { + final int length) { return PrivateCellUtil.matchingRows(left, buf, offset, length); }