Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL #1079

Merged
merged 1 commit into from
Jan 22, 2020
+150 −18
Merged

HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL #1079

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 150 additions & 18 deletions hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,23 @@
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.TimestampsFilter;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
Expand All @@ -54,7 +60,7 @@ public class TestMinVersions {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestMinVersions.class);

HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
HBaseTestingUtility hbu = new HBaseTestingUtility();
private final byte[] T0 = Bytes.toBytes("0");
private final byte[] T1 = Bytes.toBytes("1");
private final byte[] T2 = Bytes.toBytes("2");
Expand All @@ -71,8 +77,14 @@ public class TestMinVersions {
*/
@Test
public void testGetClosestBefore() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);

ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);
try {

Expand Down Expand Up @@ -121,8 +133,15 @@ public void testGetClosestBefore() throws Exception {
@Test
public void testStoreMemStore() throws Exception {
// keep 3 versions minimum
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);

ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();

HRegion region = hbu.createLocalHRegion(htd, null, null);
// 2s in the past
long ts = EnvironmentEdgeManager.currentTime() - 2000;
Expand Down Expand Up @@ -176,8 +195,14 @@ public void testStoreMemStore() throws Exception {
*/
@Test
public void testDelete() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();

HRegion region = hbu.createLocalHRegion(htd, null, null);

// 2s in the past
Expand Down Expand Up @@ -235,8 +260,13 @@ public void testDelete() throws Exception {
*/
@Test
public void testMemStore() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);

// 2s in the past
Expand Down Expand Up @@ -310,9 +340,14 @@ public void testMemStore() throws Exception {
*/
@Test
public void testBaseCase() throws Exception {
// 1 version minimum, 1000 versions maximum, ttl = 1s
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
// 2 version minimum, 1000 versions maximum, ttl = 1s
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);
try {

Expand Down Expand Up @@ -403,10 +438,23 @@ public void testBaseCase() throws Exception {
*/
@Test
public void testFilters() throws Exception {
HTableDescriptor htd =
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
HRegion region = hbu.createLocalHRegion(htd, null, null);
final byte [] c1 = COLUMNS[1];
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();

ColumnFamilyDescriptor cfd2 =
ColumnFamilyDescriptorBuilder.newBuilder(c1)
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
List<ColumnFamilyDescriptor> cfdList = new ArrayList();
cfdList.add(cfd);
cfdList.add(cfd2);

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);

// 2s in the past
long ts = EnvironmentEdgeManager.currentTime() - 2000;
Expand Down Expand Up @@ -476,12 +524,96 @@ public void testFilters() throws Exception {
}
}

@Test
public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
int ttl = 4;
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl).
setKeepDeletedCells(KeepDeletedCells.TTL).build();

TableDescriptor htd = TableDescriptorBuilder.
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();

HRegion region = hbu.createLocalHRegion(htd, null, null);

long startTS = EnvironmentEdgeManager.currentTime();
ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
injectEdge.setValue(startTS);
EnvironmentEdgeManager.injectEdge(injectEdge);

long ts = startTS - 2000;
// 1st version
Put p = new Put(T1, ts-3);
p.addColumn(c0, c0, T1);
region.put(p);

// 2nd version
p = new Put(T1, ts-2);
p.addColumn(c0, c0, T2);
region.put(p);

// 3rd version
p = new Put(T1, ts-1);
p.addColumn(c0, c0, T3);
region.put(p);

Get g;
Result r;

//check we can still see all versions before compaction
g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts);
r = region.get(g);
checkResult(r, c0, T3, T2, T1);

region.flush(true);
region.compact(true);
Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3;
Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
//check that nothing was purged yet
g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts);
r = region.get(g);
checkResult(r, c0, T3, T2, T1);

g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts -1);
r = region.get(g);
checkResult(r, c0, T2, T1);

injectEdge.incValue(ttl * 1000);

region.flush(true);
region.compact(true);

//check that after compaction (which is after TTL) that only T1 was purged
g = new Get(T1);
g.readAllVersions();
g.setTimeRange(0, ts);
r = region.get(g);
checkResult(r, c0, T3, T2);

g = new Get(T1);
g.readAllVersions();
g.setTimestamp(ts -2);
r = region.get(g);
checkResult(r, c0, T2);
}

private void checkResult(Result r, byte[] col, byte[] ... vals) {
assertEquals(r.size(), vals.length);
assertEquals(vals.length, r.size());
List<Cell> kvs = r.getColumnCells(col, col);
assertEquals(kvs.size(), vals.length);
for (int i=0;i<vals.length;i++) {
assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
String expected = Bytes.toString(vals[i]);
String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
assertTrue(expected + " was expected but doesn't match " + actual,
CellUtil.matchingValue(kvs.get(i), vals[i]));
}
}

Expand Down