Merge cockroachdb#105153

105153: admission: add metric for total tokens taken r=bananabrick a=bananabrick in the kvStoreTokenGranter Epic: none Release note: None Co-authored-by: Arjun Nair <[email protected]>
wenyihu6 · Jun 21, 2023 · 51a41bc · 51a41bc
2 parents 1a7cacf + 4fb803b
commit 51a41bc
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 0 deletions.
diff --git a/pkg/util/admission/grant_coordinator.go b/pkg/util/admission/grant_coordinator.go
@@ -54,6 +54,7 @@ type StoreGrantCoordinators struct {
 	kvIOTokensExhaustedDuration     *metric.Counter
 	kvIOTokensAvailable             *metric.Gauge
 	kvIOTokensTookWithoutPermission *metric.Counter
+	kvIOTotalTokensTaken            *metric.Counter
 
 	// These metrics are shared by WorkQueues across stores.
 	workQueueMetrics *WorkQueueMetrics
@@ -167,6 +168,7 @@ func (sgc *StoreGrantCoordinators) initGrantCoordinator(storeID roachpb.StoreID)
 		ioTokensExhaustedDurationMetric: sgc.kvIOTokensExhaustedDuration,
 		availableTokensMetrics:          sgc.kvIOTokensAvailable,
 		tookWithoutPermissionMetric:     sgc.kvIOTokensTookWithoutPermission,
+		totalTokensTaken:                sgc.kvIOTotalTokensTaken,
 	}
 	kvg.coordMu.availableIOTokens = unlimitedTokens / unloadedDuration.ticksInAdjustmentInterval()
 	kvg.coordMu.elasticDiskBWTokensAvailable = unlimitedTokens / unloadedDuration.ticksInAdjustmentInterval()
@@ -456,6 +458,7 @@ func makeStoresGrantCoordinators(
 		makeStoreRequesterFunc:          makeStoreRequester,
 		kvIOTokensExhaustedDuration:     metrics.KVIOTokensExhaustedDuration,
 		kvIOTokensTookWithoutPermission: metrics.KVIOTokensTookWithoutPermission,
+		kvIOTotalTokensTaken:            metrics.KVIOTotalTokensTaken,
 		kvIOTokensAvailable:             metrics.KVIOTokensAvailable,
 		workQueueMetrics:                storeWorkQueueMetrics,
 		onLogEntryAdmitted:              onLogEntryAdmitted,
@@ -997,6 +1000,7 @@ type GrantCoordinatorMetrics struct {
 	// TODO(banabrick): Make these metrics per store.
 	KVIOTokensExhaustedDuration     *metric.Counter
 	KVIOTokensTookWithoutPermission *metric.Counter
+	KVIOTotalTokensTaken            *metric.Counter
 	KVIOTokensAvailable             *metric.Gauge
 	SQLLeafStartUsedSlots           *metric.Gauge
 	SQLRootStartUsedSlots           *metric.Gauge
@@ -1018,6 +1022,7 @@ func makeGrantCoordinatorMetrics() GrantCoordinatorMetrics {
 		SQLLeafStartUsedSlots:           metric.NewGauge(addName(workKindString(SQLStatementLeafStartWork), usedSlots)),
 		SQLRootStartUsedSlots:           metric.NewGauge(addName(workKindString(SQLStatementRootStartWork), usedSlots)),
 		KVIOTokensTookWithoutPermission: metric.NewCounter(kvIONumIOTokensTookWithoutPermission),
+		KVIOTotalTokensTaken:            metric.NewCounter(kvIOTotalTokensTaken),
 		KVIOTokensAvailable:             metric.NewGauge(kvIOTokensAvailable),
 	}
 	return m

diff --git a/pkg/util/admission/granter.go b/pkg/util/admission/granter.go
@@ -310,6 +310,7 @@ type kvStoreTokenGranter struct {
 	ioTokensExhaustedDurationMetric *metric.Counter
 	availableTokensMetrics          *metric.Gauge
 	tookWithoutPermissionMetric     *metric.Counter
+	totalTokensTaken                *metric.Counter
 	exhaustedStart                  time.Time
 
 	// Estimation models.
@@ -392,13 +393,15 @@ func (sg *kvStoreTokenGranter) tryGetLocked(count int64, demuxHandle int8) grant
 		if sg.coordMu.availableIOTokens > 0 {
 			sg.subtractTokensLocked(count, false)
 			sg.coordMu.diskBWTokensUsed[wc] += count
+			sg.totalTokensTaken.Inc(count)
 			return grantSuccess
 		}
 	case admissionpb.ElasticWorkClass:
 		if sg.coordMu.elasticDiskBWTokensAvailable > 0 && sg.coordMu.availableIOTokens > 0 {
 			sg.coordMu.elasticDiskBWTokensAvailable -= count
 			sg.subtractTokensLocked(count, false)
 			sg.coordMu.diskBWTokensUsed[wc] += count
+			sg.totalTokensTaken.Inc(count)
 			return grantSuccess
 		}
 	}
@@ -430,6 +433,7 @@ func (sg *kvStoreTokenGranter) tookWithoutPermissionLocked(count int64, demuxHan
 	wc := admissionpb.WorkClass(demuxHandle)
 	sg.subtractTokensLocked(count, false)
 	sg.tookWithoutPermissionMetric.Inc(count)
+	sg.totalTokensTaken.Inc(count)
 	if wc == admissionpb.ElasticWorkClass {
 		sg.coordMu.elasticDiskBWTokensAvailable -= count
 	}
@@ -684,6 +688,12 @@ var (
 		Measurement: "Tokens",
 		Unit:        metric.Unit_COUNT,
 	}
+	kvIOTotalTokensTaken = metric.Metadata{
+		Name:        "admission.granter.io_tokens_taken.kv",
+		Help:        "Total number of tokens taken",
+		Measurement: "Tokens",
+		Unit:        metric.Unit_COUNT,
+	}
 	kvIOTokensAvailable = metric.Metadata{
 		Name:        "admission.granter.io_tokens_available.kv",
 		Help:        "Number of tokens available",

diff --git a/pkg/util/admission/granter_test.go b/pkg/util/admission/granter_test.go
@@ -140,6 +140,7 @@ func TestGranterBasic(t *testing.T) {
 				kvIOTokensExhaustedDuration:     metrics.KVIOTokensExhaustedDuration,
 				kvIOTokensAvailable:             metrics.KVIOTokensAvailable,
 				kvIOTokensTookWithoutPermission: metrics.KVIOTokensTookWithoutPermission,
+				kvIOTotalTokensTaken:            metrics.KVIOTotalTokensTaken,
 				workQueueMetrics:                workQueueMetrics,
 				disableTickerForTesting:         true,
 			}