From 4fb803b556751939340470b801328c95c275eb34 Mon Sep 17 00:00:00 2001 From: Arjun Nair Date: Mon, 19 Jun 2023 15:58:48 -0400 Subject: [PATCH] admission: add metric for total tokens taken in the kvStoreTokenGranter Epic: none Release note: None --- pkg/util/admission/grant_coordinator.go | 5 +++++ pkg/util/admission/granter.go | 10 ++++++++++ pkg/util/admission/granter_test.go | 1 + 3 files changed, 16 insertions(+) diff --git a/pkg/util/admission/grant_coordinator.go b/pkg/util/admission/grant_coordinator.go index e0bb3fe4b724..cee01b1e6c68 100644 --- a/pkg/util/admission/grant_coordinator.go +++ b/pkg/util/admission/grant_coordinator.go @@ -54,6 +54,7 @@ type StoreGrantCoordinators struct { kvIOTokensExhaustedDuration *metric.Counter kvIOTokensAvailable *metric.Gauge kvIOTokensTookWithoutPermission *metric.Counter + kvIOTotalTokensTaken *metric.Counter // These metrics are shared by WorkQueues across stores. workQueueMetrics *WorkQueueMetrics @@ -167,6 +168,7 @@ func (sgc *StoreGrantCoordinators) initGrantCoordinator(storeID roachpb.StoreID) ioTokensExhaustedDurationMetric: sgc.kvIOTokensExhaustedDuration, availableTokensMetrics: sgc.kvIOTokensAvailable, tookWithoutPermissionMetric: sgc.kvIOTokensTookWithoutPermission, + totalTokensTaken: sgc.kvIOTotalTokensTaken, } kvg.coordMu.availableIOTokens = unlimitedTokens / unloadedDuration.ticksInAdjustmentInterval() kvg.coordMu.elasticDiskBWTokensAvailable = unlimitedTokens / unloadedDuration.ticksInAdjustmentInterval() @@ -456,6 +458,7 @@ func makeStoresGrantCoordinators( makeStoreRequesterFunc: makeStoreRequester, kvIOTokensExhaustedDuration: metrics.KVIOTokensExhaustedDuration, kvIOTokensTookWithoutPermission: metrics.KVIOTokensTookWithoutPermission, + kvIOTotalTokensTaken: metrics.KVIOTotalTokensTaken, kvIOTokensAvailable: metrics.KVIOTokensAvailable, workQueueMetrics: storeWorkQueueMetrics, onLogEntryAdmitted: onLogEntryAdmitted, @@ -997,6 +1000,7 @@ type GrantCoordinatorMetrics struct { // TODO(banabrick): Make these metrics per store. KVIOTokensExhaustedDuration *metric.Counter KVIOTokensTookWithoutPermission *metric.Counter + KVIOTotalTokensTaken *metric.Counter KVIOTokensAvailable *metric.Gauge SQLLeafStartUsedSlots *metric.Gauge SQLRootStartUsedSlots *metric.Gauge @@ -1018,6 +1022,7 @@ func makeGrantCoordinatorMetrics() GrantCoordinatorMetrics { SQLLeafStartUsedSlots: metric.NewGauge(addName(workKindString(SQLStatementLeafStartWork), usedSlots)), SQLRootStartUsedSlots: metric.NewGauge(addName(workKindString(SQLStatementRootStartWork), usedSlots)), KVIOTokensTookWithoutPermission: metric.NewCounter(kvIONumIOTokensTookWithoutPermission), + KVIOTotalTokensTaken: metric.NewCounter(kvIOTotalTokensTaken), KVIOTokensAvailable: metric.NewGauge(kvIOTokensAvailable), } return m diff --git a/pkg/util/admission/granter.go b/pkg/util/admission/granter.go index e49c558c1413..38d19eaa4dc7 100644 --- a/pkg/util/admission/granter.go +++ b/pkg/util/admission/granter.go @@ -310,6 +310,7 @@ type kvStoreTokenGranter struct { ioTokensExhaustedDurationMetric *metric.Counter availableTokensMetrics *metric.Gauge tookWithoutPermissionMetric *metric.Counter + totalTokensTaken *metric.Counter exhaustedStart time.Time // Estimation models. @@ -392,6 +393,7 @@ func (sg *kvStoreTokenGranter) tryGetLocked(count int64, demuxHandle int8) grant if sg.coordMu.availableIOTokens > 0 { sg.subtractTokensLocked(count, false) sg.coordMu.diskBWTokensUsed[wc] += count + sg.totalTokensTaken.Inc(count) return grantSuccess } case admissionpb.ElasticWorkClass: @@ -399,6 +401,7 @@ func (sg *kvStoreTokenGranter) tryGetLocked(count int64, demuxHandle int8) grant sg.coordMu.elasticDiskBWTokensAvailable -= count sg.subtractTokensLocked(count, false) sg.coordMu.diskBWTokensUsed[wc] += count + sg.totalTokensTaken.Inc(count) return grantSuccess } } @@ -430,6 +433,7 @@ func (sg *kvStoreTokenGranter) tookWithoutPermissionLocked(count int64, demuxHan wc := admissionpb.WorkClass(demuxHandle) sg.subtractTokensLocked(count, false) sg.tookWithoutPermissionMetric.Inc(count) + sg.totalTokensTaken.Inc(count) if wc == admissionpb.ElasticWorkClass { sg.coordMu.elasticDiskBWTokensAvailable -= count } @@ -684,6 +688,12 @@ var ( Measurement: "Tokens", Unit: metric.Unit_COUNT, } + kvIOTotalTokensTaken = metric.Metadata{ + Name: "admission.granter.io_tokens_taken.kv", + Help: "Total number of tokens taken", + Measurement: "Tokens", + Unit: metric.Unit_COUNT, + } kvIOTokensAvailable = metric.Metadata{ Name: "admission.granter.io_tokens_available.kv", Help: "Number of tokens available", diff --git a/pkg/util/admission/granter_test.go b/pkg/util/admission/granter_test.go index f2493802a311..f4e6ea0545c1 100644 --- a/pkg/util/admission/granter_test.go +++ b/pkg/util/admission/granter_test.go @@ -140,6 +140,7 @@ func TestGranterBasic(t *testing.T) { kvIOTokensExhaustedDuration: metrics.KVIOTokensExhaustedDuration, kvIOTokensAvailable: metrics.KVIOTokensAvailable, kvIOTokensTookWithoutPermission: metrics.KVIOTokensTookWithoutPermission, + kvIOTotalTokensTaken: metrics.KVIOTotalTokensTaken, workQueueMetrics: workQueueMetrics, disableTickerForTesting: true, }