Skip to content

Commit

Permalink
HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL (apac…
Browse files Browse the repository at this point in the history 
…he#1079)

Signed-off-by: stack <[email protected]>
Signed-off-by: Viraj Jasani <[email protected]>
  • Loading branch information
@gjacoby126 @thangTang
gjacoby126 authored and thangTang committed Apr 16, 2020
1 parent 379a217 commit bdc806f
Showing 1 changed file with 150 additions and 18 deletions.
168 changes: 150 additions & 18 deletions hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,23 @@
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.TimestampsFilter;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
Expand All @@ -54,7 +60,7 @@ public class TestMinVersions {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestMinVersions.class);

HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
HBaseTestingUtility hbu = new HBaseTestingUtility();
private final byte[] T0 = Bytes.toBytes("0");
private final byte[] T1 = Bytes.toBytes("1");
private final byte[] T2 = Bytes.toBytes("2");
Expand All @@ -71,8 +77,14 @@ public class TestMinVersions {
*/
@Test
public void testGetClosestBefore() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);

ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);
try {

Expand Down Expand Up @@ -121,8 +133,15 @@ public void testGetClosestBefore() throws Exception {
@Test
public void testStoreMemStore() throws Exception {
// keep 3 versions minimum
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);

ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();

HRegion region = hbu.createLocalHRegion(htd, null, null);
// 2s in the past
long ts = EnvironmentEdgeManager.currentTime() - 2000;
Expand Down Expand Up @@ -176,8 +195,14 @@ public void testStoreMemStore() throws Exception {
*/
@Test
public void testDelete() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();

HRegion region = hbu.createLocalHRegion(htd, null, null);

// 2s in the past
Expand Down Expand Up @@ -235,8 +260,13 @@ public void testDelete() throws Exception {
*/
@Test
public void testMemStore() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);

// 2s in the past
Expand Down Expand Up @@ -310,9 +340,14 @@ public void testMemStore() throws Exception {
*/
@Test
public void testBaseCase() throws Exception {
// 1 version minimum, 1000 versions maximum, ttl = 1s
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
// 2 version minimum, 1000 versions maximum, ttl = 1s
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);
try {

Expand Down Expand Up @@ -403,10 +438,23 @@ public void testBaseCase() throws Exception {
*/
@Test
public void testFilters() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
HRegion region = hbu.createLocalHRegion(htd, null, null);
final byte [] c1 = COLUMNS[1];
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

ColumnFamilyDescriptor cfd2 =
ColumnFamilyDescriptorBuilder.newBuilder(c1)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
List<ColumnFamilyDescriptor> cfdList = new ArrayList();
cfdList.add(cfd);
cfdList.add(cfd2);

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);

// 2s in the past
long ts = EnvironmentEdgeManager.currentTime() - 2000;
Expand Down Expand Up @@ -476,12 +524,96 @@ public void testFilters() throws Exception {
}
}

@Test
public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
int ttl = 4;
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl).
setKeepDeletedCells(KeepDeletedCells.TTL).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();

HRegion region = hbu.createLocalHRegion(htd, null, null);

long startTS = EnvironmentEdgeManager.currentTime();
ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
injectEdge.setValue(startTS);
EnvironmentEdgeManager.injectEdge(injectEdge);

long ts = startTS - 2000;
// 1st version
Put p = new Put(T1, ts-3);
p.addColumn(c0, c0, T1);
region.put(p);

// 2nd version
p = new Put(T1, ts-2);
p.addColumn(c0, c0, T2);
region.put(p);

// 3rd version
p = new Put(T1, ts-1);
p.addColumn(c0, c0, T3);
region.put(p);

Get g;
Result r;

//check we can still see all versions before compaction
g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts);
r = region.get(g);
checkResult(r, c0, T3, T2, T1);

region.flush(true);
region.compact(true);
Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3;
Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
//check that nothing was purged yet
g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts);
r = region.get(g);
checkResult(r, c0, T3, T2, T1);

g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts -1);
r = region.get(g);
checkResult(r, c0, T2, T1);

injectEdge.incValue(ttl * 1000);

region.flush(true);
region.compact(true);

//check that after compaction (which is after TTL) that only T1 was purged
g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts);
r = region.get(g);
checkResult(r, c0, T3, T2);

g = new Get(T1);
g.readAllVersions();
g.setTimestamp(ts -2);
r = region.get(g);
checkResult(r, c0, T2);
}

private void checkResult(Result r, byte[] col, byte[] ... vals) {
assertEquals(r.size(), vals.length);
assertEquals(vals.length, r.size());
List<Cell> kvs = r.getColumnCells(col, col);
assertEquals(kvs.size(), vals.length);
for (int i=0;i<vals.length;i++) {
assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
String expected = Bytes.toString(vals[i]);
String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
assertTrue(expected + " was expected but doesn't match " + actual,
CellUtil.matchingValue(kvs.get(i), vals[i]));
}
}

Expand Down

0 comments on commit bdc806f

Please sign in to comment.