diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java index 451fc3b800b1..84ec85b54114 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java @@ -70,6 +70,18 @@ public interface MetricsHeapMemoryManagerSource extends BaseSource { */ void setCurMemStoreSizeGauge(long memStoreSize); + /** + * Set the current global memstore on-heap size used gauge + * @param memStoreOnHeapSize the current memory usage in memstore on-heap, in bytes. + */ + void setCurMemStoreOnHeapSizeGauge(long memStoreOnHeapSize); + + /** + * Set the current global memstore off-heap size used gauge + * @param memStoreOffHeapSize the current memory usage in memstore off-heap, in bytes. + */ + void setCurMemStoreOffHeapSizeGauge(long memStoreOffHeapSize); + /** * Update the increase/decrease memstore size histogram * @param memStoreDeltaSize the tuning result of memstore. @@ -118,6 +130,13 @@ public interface MetricsHeapMemoryManagerSource extends BaseSource { String UNBLOCKED_FLUSH_GAUGE_DESC = "Gauge for the unblocked flush count before tuning"; String MEMSTORE_SIZE_GAUGE_NAME = "memStoreSize"; String MEMSTORE_SIZE_GAUGE_DESC = "Global MemStore used in bytes by the RegionServer"; + String MEMSTORE_ONHEAP_SIZE_GAUGE_NAME = "memStoreOnHeapSize"; + String MEMSTORE_ONHEAP_SIZE_GAUGE_DESC = + "Global MemStore On-heap size in bytes by the RegionServer"; + String MEMSTORE_OFFHEAP_SIZE_GAUGE_NAME = "memStoreOffHeapSize"; + String MEMSTORE_OFFHEAP_SIZE_GAUGE_DESC = + "Global MemStore Off-heap size in bytes by the RegionServer"; + String BLOCKCACHE_SIZE_GAUGE_NAME = "blockCacheSize"; String BLOCKCACHE_SIZE_GAUGE_DESC = "BlockCache used in bytes by the RegionServer"; diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java index c2e8d329143c..e8967246dd75 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java @@ -41,6 +41,8 @@ public class MetricsHeapMemoryManagerSourceImpl extends BaseSourceImpl private final MutableGaugeLong blockedFlushGauge; private final MutableGaugeLong unblockedFlushGauge; private final MutableGaugeLong memStoreSizeGauge; + private final MutableGaugeLong memStoreOnHeapSizeGauge; + private final MutableGaugeLong memStoreOffHeapSizeGauge; private final MutableGaugeLong blockCacheSizeGauge; private final MutableFastCounter doNothingCounter; @@ -75,6 +77,10 @@ public MetricsHeapMemoryManagerSourceImpl(String metricsName, String metricsDesc getMetricsRegistry().newGauge(UNBLOCKED_FLUSH_GAUGE_NAME, UNBLOCKED_FLUSH_GAUGE_DESC, 0L); memStoreSizeGauge = getMetricsRegistry().newGauge(MEMSTORE_SIZE_GAUGE_NAME, MEMSTORE_SIZE_GAUGE_DESC, 0L); + memStoreOnHeapSizeGauge = getMetricsRegistry().newGauge(MEMSTORE_ONHEAP_SIZE_GAUGE_NAME, + MEMSTORE_ONHEAP_SIZE_GAUGE_DESC, 0L); + memStoreOffHeapSizeGauge = getMetricsRegistry().newGauge(MEMSTORE_OFFHEAP_SIZE_GAUGE_NAME, + MEMSTORE_OFFHEAP_SIZE_GAUGE_DESC, 0L); blockCacheSizeGauge = getMetricsRegistry().newGauge(BLOCKCACHE_SIZE_GAUGE_NAME, BLOCKCACHE_SIZE_GAUGE_DESC, 0L); @@ -111,6 +117,16 @@ public void setCurMemStoreSizeGauge(long memstoreSize) { memStoreSizeGauge.set(memstoreSize); } + @Override + public void setCurMemStoreOnHeapSizeGauge(long memstoreOnHeapSize) { + memStoreOnHeapSizeGauge.set(memstoreOnHeapSize); + } + + @Override + public void setCurMemStoreOffHeapSizeGauge(long memstoreOffHeapSize) { + memStoreOffHeapSizeGauge.set(memstoreOffHeapSize); + } + @Override public void updateMemStoreDeltaSizeHistogram(int memStoreDeltaSize) { if (memStoreDeltaSize >= 0) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index 7a72b9af41cf..fe2737b0a7d2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -317,12 +317,15 @@ private void tune() { unblockedFlushCnt = unblockedFlushCount.getAndSet(0); tunerContext.setUnblockedFlushCount(unblockedFlushCnt); metricsHeapMemoryManager.updateUnblockedFlushCount(unblockedFlushCnt); - // TODO : add support for offheap metrics tunerContext.setCurBlockCacheUsed((float) blockCache.getCurrentSize() / maxHeapSize); metricsHeapMemoryManager.setCurBlockCacheSizeGauge(blockCache.getCurrentSize()); + long globalMemstoreDataSize = regionServerAccounting.getGlobalMemStoreDataSize(); long globalMemstoreHeapSize = regionServerAccounting.getGlobalMemStoreHeapSize(); + long globalMemStoreOffHeapSize = regionServerAccounting.getGlobalMemStoreOffHeapSize(); tunerContext.setCurMemStoreUsed((float) globalMemstoreHeapSize / maxHeapSize); - metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreHeapSize); + metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreDataSize); + metricsHeapMemoryManager.setCurMemStoreOnHeapSizeGauge(globalMemstoreHeapSize); + metricsHeapMemoryManager.setCurMemStoreOffHeapSizeGauge(globalMemStoreOffHeapSize); tunerContext.setCurBlockCacheSize(blockCachePercent); tunerContext.setCurMemStoreSize(globalMemStorePercent); TunerResult result = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java index e781bddb2274..2f3e43fccc93 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java @@ -73,6 +73,22 @@ public void setCurMemStoreSizeGauge(final long memStoreSize) { source.setCurMemStoreSizeGauge(memStoreSize); } + /** + * Set the current global memstore on-heap size gauge + * @param memStoreOnHeapSize the current memory on-heap size in memstore, in bytes. + */ + public void setCurMemStoreOnHeapSizeGauge(final long memStoreOnHeapSize) { + source.setCurMemStoreOnHeapSizeGauge(memStoreOnHeapSize); + } + + /** + * Set the current global memstore off-heap size gauge + * @param memStoreOffHeapSize the current memory off-heap size in memstore, in bytes. + */ + public void setCurMemStoreOffHeapSizeGauge(final long memStoreOffHeapSize) { + source.setCurMemStoreOffHeapSizeGauge(memStoreOffHeapSize); + } + /** * Update the increase/decrease memstore size histogram * @param memStoreDeltaSize the tuning result of memstore. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java index c1706c995e7e..b26611cb4265 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java @@ -121,12 +121,12 @@ public long getGlobalMemStoreDataSize() { return globalMemStoreDataSize.sum(); } - /** Returns the global memstore heap size in the RegionServer */ + /** Returns the global memstore on-heap size in the RegionServer */ public long getGlobalMemStoreHeapSize() { return this.globalMemStoreHeapSize.sum(); } - /** Returns the global memstore heap size in the RegionServer */ + /** Returns the global memstore off-heap size in the RegionServer */ public long getGlobalMemStoreOffHeapSize() { return this.globalMemStoreOffHeapSize.sum(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java index 2d6a47fc2122..974d6485377e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java @@ -74,11 +74,15 @@ public void testGauge() { hmm.updateBlockedFlushCount(200); hmm.updateUnblockedFlushCount(50); hmm.setCurMemStoreSizeGauge(256 * 1024 * 1024); + hmm.setCurMemStoreOnHeapSizeGauge(512 * 1024 * 1024); + hmm.setCurMemStoreOffHeapSizeGauge(128 * 1024 * 1024); hmm.setCurBlockCacheSizeGauge(100 * 1024 * 1024); HELPER.assertGauge("blockedFlushGauge", 200, source); HELPER.assertGauge("unblockedFlushGauge", 50, source); HELPER.assertGauge("memStoreSize", 256 * 1024 * 1024, source); + HELPER.assertGauge("memStoreOnHeapSize", 512 * 1024 * 1024, source); + HELPER.assertGauge("memStoreOffHeapSize", 128 * 1024 * 1024, source); HELPER.assertGauge("blockCacheSize", 100 * 1024 * 1024, source); } }