Skip to content

Commit

Permalink
HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_re…
Browse files Browse the repository at this point in the history 
…gions property to limit max number of regions in a merge request for merge normalization

Signed-off-by: Nick Dimiduk <[email protected]>
  • Loading branch information
@ndimiduk
Rahul Kumar authored and ndimiduk committed Sep 21, 2023
1 parent 82c354e commit dad1f70
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 13 deletions.
6 changes: 6 additions & 0 deletions hbase-common/src/main/resources/hbase-default.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,12 @@ possible configurations would overwhelm and obscure the important.
<description>The minimum size for a region to be considered for a merge, in whole
MBs.</description>
</property>
<property>
<name>hbase.normalizer.merge.merge_request_max_number_of_regions</name>
<value>50</value>
<description>The maximum number of region count in a merge request for merge
normalization.</description>
</property>
<property>
<name>hbase.table.normalization.enabled</name>
<value>false</value>
Expand Down
55 changes: 42 additions & 13 deletions ...erver/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3;
static final String MERGE_MIN_REGION_SIZE_MB_KEY = "hbase.normalizer.merge.min_region_size.mb";
static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 0;
static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY =
"hbase.normalizer.merge.merge_request_max_number_of_regions";
static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100;

private MasterServices masterServices;
private NormalizerConfiguration normalizerConfiguration;
Expand Down Expand Up @@ -138,6 +141,16 @@ private static long parseMergeMinRegionSizeMb(final Configuration conf) {
return settledValue;
}

private static long parseMergeRequestMaxNumberOfRegionsCount(final Configuration conf) {
final long parsedValue = conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT);
final long settledValue = Math.max(2, parsedValue);
if (parsedValue != settledValue) {
warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, parsedValue, settledValue);
}
return settledValue;
}

private static <T> void warnInvalidValue(final String key, final T parsedValue,
final T settledValue) {
LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key, parsedValue,
Expand Down Expand Up @@ -186,6 +199,10 @@ public long getMergeMinRegionSizeMb() {
return normalizerConfiguration.getMergeMinRegionSizeMb();
}

public long getMergeRequestMaxNumberOfRegionsCount() {
return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount();
}

@Override
public void setMasterServices(final MasterServices masterServices) {
this.masterServices = masterServices;
Expand Down Expand Up @@ -382,19 +399,21 @@ private List<NormalizationPlan> computeMergeNormalizationPlans(final NormalizeCo
break;
}
if (
rangeMembers.isEmpty() // when there are no range members, seed the range with whatever
// we have. this way we're prepared in case the next region is
// 0-size.
|| (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) // when there is only one
// region and the size is 0,
// seed the range with
// whatever we have.
|| regionSizeMb == 0 // always add an empty region to the current range.
|| (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
) { // add the current region
// to the range when
// there's capacity
// remaining.
// when there are no range members, seed the range with whatever we have. this way we're
// prepared in case the next region is 0-size.
rangeMembers.isEmpty()
// when there is only one region and the size is 0, seed the range with whatever we
// have.
|| (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0)
// add an empty region to the current range only if it doesn't exceed max merge request
// region count
|| (regionSizeMb == 0 && rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount())
// add region if current range region size is less than avg region size of table
// and current range doesn't exceed max merge request region count
|| ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
&& (rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount()))
) {
// add the current region to the range when there's capacity remaining.
rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb));
sumRangeMembersSizeMb += regionSizeMb;
continue;
Expand Down Expand Up @@ -502,6 +521,7 @@ private static final class NormalizerConfiguration {
private final int mergeMinRegionCount;
private final Period mergeMinRegionAge;
private final long mergeMinRegionSizeMb;
private final long mergeRequestMaxNumberOfRegionsCount;
private final long cumulativePlansSizeLimitMb;

private NormalizerConfiguration() {
Expand All @@ -511,6 +531,7 @@ private NormalizerConfiguration() {
mergeMinRegionCount = DEFAULT_MERGE_MIN_REGION_COUNT;
mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS);
mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB;
mergeRequestMaxNumberOfRegionsCount = DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT;
cumulativePlansSizeLimitMb = DEFAULT_CUMULATIVE_SIZE_LIMIT_MB;
}

Expand All @@ -522,6 +543,7 @@ private NormalizerConfiguration(final Configuration conf,
mergeMinRegionCount = parseMergeMinRegionCount(conf);
mergeMinRegionAge = parseMergeMinRegionAge(conf);
mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf);
mergeRequestMaxNumberOfRegionsCount = parseMergeRequestMaxNumberOfRegionsCount(conf);
cumulativePlansSizeLimitMb =
conf.getLong(CUMULATIVE_SIZE_LIMIT_MB_KEY, DEFAULT_CUMULATIVE_SIZE_LIMIT_MB);
logConfigurationUpdated(SPLIT_ENABLED_KEY, currentConfiguration.isSplitEnabled(),
Expand All @@ -534,6 +556,9 @@ private NormalizerConfiguration(final Configuration conf,
currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge);
logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY,
currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb);
logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(),
mergeRequestMaxNumberOfRegionsCount);
}

public Configuration getConf() {
Expand Down Expand Up @@ -597,6 +622,10 @@ public long getMergeMinRegionSizeMb(NormalizeContext context) {
return mergeMinRegionSizeMb;
}

public long getMergeRequestMaxNumberOfRegionsCount() {
return mergeRequestMaxNumberOfRegionsCount;
}

private long getCumulativePlansSizeLimitMb() {
return cumulativePlansSizeLimitMb;
}
Expand Down
36 changes: 36 additions & 0 deletions ...r/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_COUNT_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
import static org.hamcrest.MatcherAssert.assertThat;
Expand Down Expand Up @@ -503,6 +504,41 @@ public void testHonorsMergeMinRegionSizeInTD() {
assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
}

@Test
public void testHonorsMergeRequestMaxNumberOfRegionsCount() {
conf.setBoolean(SPLIT_ENABLED_KEY, false);
conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3);
final TableName tableName = name.getTableName();
final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0);
setupMocksForNormalizer(regionSizes, regionInfos);
assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount());
List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
assertThat(plans,
contains(
new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
.addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(),
new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1)
.addTarget(regionInfos.get(4), 0).build()));
}

@Test
public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() {
conf.setBoolean(SPLIT_ENABLED_KEY, false);
conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
final TableName tableName = name.getTableName();
final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0);
setupMocksForNormalizer(regionSizes, regionInfos);
assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount());
List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
.addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build()));
}

@Test
public void testMergeEmptyRegions0() {
conf.setBoolean(SPLIT_ENABLED_KEY, false);
Expand Down

0 comments on commit dad1f70

Please sign in to comment.