From bdc806f2975c740ce3426409e969b7d38243b426 Mon Sep 17 00:00:00 2001 From: Geoffrey Jacoby Date: Wed, 22 Jan 2020 00:07:15 -0800 Subject: [PATCH] HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL (#1079) Signed-off-by: stack Signed-off-by: Viraj Jasani --- .../hbase/regionserver/TestMinVersions.java | 168 ++++++++++++++++-- 1 file changed, 150 insertions(+), 18 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java index 0a33c58c7632..84eb25ea49b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java @@ -27,17 +27,23 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeepDeletedCells; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.filter.TimestampsFilter; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.junit.Assert; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -54,7 +60,7 @@ public class TestMinVersions { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestMinVersions.class); - HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU(); + HBaseTestingUtility hbu = new HBaseTestingUtility(); private final byte[] T0 = Bytes.toBytes("0"); private final byte[] T1 = Bytes.toBytes("1"); private final byte[] T2 = Bytes.toBytes("2"); @@ -71,8 +77,14 @@ public class TestMinVersions { */ @Test public void testGetClosestBefore() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE); + + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(1).setMaxVersions(1000).setTimeToLive(1). + setKeepDeletedCells(KeepDeletedCells.FALSE).build(); + + TableDescriptor htd = TableDescriptorBuilder. + newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); HRegion region = hbu.createLocalHRegion(htd, null, null); try { @@ -121,8 +133,15 @@ public void testGetClosestBefore() throws Exception { @Test public void testStoreMemStore() throws Exception { // keep 3 versions minimum - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); + + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1). + setKeepDeletedCells(KeepDeletedCells.FALSE).build(); + + TableDescriptor htd = TableDescriptorBuilder. + newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); + HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past long ts = EnvironmentEdgeManager.currentTime() - 2000; @@ -176,8 +195,14 @@ public void testStoreMemStore() throws Exception { */ @Test public void testDelete() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1). + setKeepDeletedCells(KeepDeletedCells.FALSE).build(); + + TableDescriptor htd = TableDescriptorBuilder. + newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); + HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past @@ -235,8 +260,13 @@ public void testDelete() throws Exception { */ @Test public void testMemStore() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). + setKeepDeletedCells(KeepDeletedCells.FALSE).build(); + + TableDescriptor htd = TableDescriptorBuilder. + newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past @@ -310,9 +340,14 @@ public void testMemStore() throws Exception { */ @Test public void testBaseCase() throws Exception { - // 1 version minimum, 1000 versions maximum, ttl = 1s - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); + // 2 version minimum, 1000 versions maximum, ttl = 1s + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). + setKeepDeletedCells(KeepDeletedCells.FALSE).build(); + + TableDescriptor htd = TableDescriptorBuilder. + newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); HRegion region = hbu.createLocalHRegion(htd, null, null); try { @@ -403,10 +438,23 @@ public void testBaseCase() throws Exception { */ @Test public void testFilters() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); - HRegion region = hbu.createLocalHRegion(htd, null, null); final byte [] c1 = COLUMNS[1]; + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). + setKeepDeletedCells(KeepDeletedCells.FALSE).build(); + + ColumnFamilyDescriptor cfd2 = + ColumnFamilyDescriptorBuilder.newBuilder(c1) + .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). + setKeepDeletedCells(KeepDeletedCells.FALSE).build(); + List cfdList = new ArrayList(); + cfdList.add(cfd); + cfdList.add(cfd2); + + TableDescriptor htd = TableDescriptorBuilder. + newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build(); + HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past long ts = EnvironmentEdgeManager.currentTime() - 2000; @@ -476,12 +524,96 @@ public void testFilters() throws Exception { } } + @Test + public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception { + int ttl = 4; + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl). + setKeepDeletedCells(KeepDeletedCells.TTL).build(); + + TableDescriptor htd = TableDescriptorBuilder. + newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); + + HRegion region = hbu.createLocalHRegion(htd, null, null); + + long startTS = EnvironmentEdgeManager.currentTime(); + ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge(); + injectEdge.setValue(startTS); + EnvironmentEdgeManager.injectEdge(injectEdge); + + long ts = startTS - 2000; + // 1st version + Put p = new Put(T1, ts-3); + p.addColumn(c0, c0, T1); + region.put(p); + + // 2nd version + p = new Put(T1, ts-2); + p.addColumn(c0, c0, T2); + region.put(p); + + // 3rd version + p = new Put(T1, ts-1); + p.addColumn(c0, c0, T3); + region.put(p); + + Get g; + Result r; + + //check we can still see all versions before compaction + g = new Get(T1); + g.readAllVersions(); + g.setTimeRange(0, ts); + r = region.get(g); + checkResult(r, c0, T3, T2, T1); + + region.flush(true); + region.compact(true); + Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime()); + long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3; + Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000)); + //check that nothing was purged yet + g = new Get(T1); + g.readAllVersions(); + g.setTimeRange(0, ts); + r = region.get(g); + checkResult(r, c0, T3, T2, T1); + + g = new Get(T1); + g.readAllVersions(); + g.setTimeRange(0, ts -1); + r = region.get(g); + checkResult(r, c0, T2, T1); + + injectEdge.incValue(ttl * 1000); + + region.flush(true); + region.compact(true); + + //check that after compaction (which is after TTL) that only T1 was purged + g = new Get(T1); + g.readAllVersions(); + g.setTimeRange(0, ts); + r = region.get(g); + checkResult(r, c0, T3, T2); + + g = new Get(T1); + g.readAllVersions(); + g.setTimestamp(ts -2); + r = region.get(g); + checkResult(r, c0, T2); + } + private void checkResult(Result r, byte[] col, byte[] ... vals) { - assertEquals(r.size(), vals.length); + assertEquals(vals.length, r.size()); List kvs = r.getColumnCells(col, col); assertEquals(kvs.size(), vals.length); for (int i=0;i