Skip to content

Commit

Permalink
HBASE-28390 WAL value compression fails for cells with large values (#…
Browse files Browse the repository at this point in the history 
…5696)

Signed-off-by: Andrew Purtell <[email protected]>
  • Loading branch information
@bbeaudreault
bbeaudreault authored Feb 23, 2024
1 parent 8c989c9 commit 0ea11f2
Show file tree
Hide file tree
Showing 14 changed files with 55 additions and 156 deletions.
13 changes: 0 additions & 13 deletions ...rc/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -60,10 +53,4 @@ public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
14 changes: 0 additions & 14 deletions ...rc/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLzo extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionLzo extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLzo.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -59,11 +52,4 @@ public static void setUpBeforeClass() throws Exception {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
13 changes: 0 additions & 13 deletions ...test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionSnappy extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionSnappy.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -60,10 +53,4 @@ public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
13 changes: 0 additions & 13 deletions ...c/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionZstd extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionZstd extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionZstd.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -60,10 +53,4 @@ public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
13 changes: 0 additions & 13 deletions ...li/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionBrotli extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionBrotli extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionBrotli.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -60,10 +53,4 @@ public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
14 changes: 0 additions & 14 deletions ...sion-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -59,11 +52,4 @@ public static void setUpBeforeClass() throws Exception {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
13 changes: 0 additions & 13 deletions ...py/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -31,10 +30,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionSnappy extends CompressedWALTestBase {
Expand All @@ -43,9 +39,6 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionSnappy.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
Expand All @@ -63,10 +56,4 @@ public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
13 changes: 0 additions & 13 deletions ...ssion-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLzma extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionLzma extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLzma.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -60,10 +53,4 @@ public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
14 changes: 0 additions & 14 deletions ...n-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -29,10 +28,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;

@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionZstd extends CompressedWALTestBase {
Expand All @@ -41,9 +37,6 @@ public class TestWALCompressionZstd extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionZstd.class);

@Rule
public TestName name = new TestName();

@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -59,11 +52,4 @@ public static void setUpBeforeClass() throws Exception {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void test() throws Exception {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}

}
17 changes: 17 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.regionserver.wal;

import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
Expand Down Expand Up @@ -129,9 +130,25 @@ public void decompress(InputStream in, int inLength, byte[] outArray, int outOff
} else {
lowerIn.reset(in, inLength);
IOUtils.readFully(compressedIn, outArray, outOffset, outLength);
// if the uncompressed size was larger than the configured buffer size for the codec,
// the BlockCompressorStream will have left an extra 4 bytes hanging. This represents a size
// for the next segment, and it should be 0. See HBASE-28390
if (lowerIn.available() == 4) {
int remaining = rawReadInt(lowerIn);
assert remaining == 0;
}
}
}

private int rawReadInt(InputStream in) throws IOException {
int b1 = in.read();
int b2 = in.read();
int b3 = in.read();
int b4 = in.read();
if ((b1 | b2 | b3 | b4) < 0) throw new EOFException();
return ((b1 << 24) + (b2 << 16) + (b3 << 8) + (b4 << 0));
}

public void clear() {
if (compressedOut != null) {
try {
Expand Down
Loading

0 comments on commit 0ea11f2

Please sign in to comment.