metrics(ticdc): fix slowest table metrics in scheduler (pingcap#10484) (

pingcap#10500) close pingcap#10354, close pingcap#10449, close pingcap#10482
ti-chi-bot · Feb 26, 2024 · ae0e285 · ae0e285
1 parent da5ba90
commit ae0e285
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 4 deletions.
diff --git a/cdc/owner/changefeed.go b/cdc/owner/changefeed.go
@@ -708,6 +708,7 @@ func (c *changefeed) initMetrics() {
 
 // releaseResources is idempotent.
 func (c *changefeed) releaseResources(ctx cdcContext.Context) {
+	c.cleanupMetrics()
 	if c.isReleased {
 		return
 	}
@@ -745,7 +746,6 @@ func (c *changefeed) releaseResources(ctx cdcContext.Context) {
 		_ = c.downstreamObserver.Close()
 	}
 
-	c.cleanupMetrics()
 	c.schema = nil
 	c.barriers = nil
 	c.initialized = false

diff --git a/cdc/scheduler/internal/v3/replication/replication_manager.go b/cdc/scheduler/internal/v3/replication/replication_manager.go
@@ -591,6 +591,7 @@ func (r *Manager) AdvanceCheckpoint(
 
 	r.slowestPuller = tablepb.Span{}
 	r.slowestSink = tablepb.Span{}
+	resolvedTsOfSlowestSink := model.Ts(math.MaxUint64)
 
 	watermark = schedulepb.Watermark{
 		CheckpointTs:     math.MaxUint64,
@@ -625,9 +626,12 @@ func (r *Manager) AdvanceCheckpoint(
 				}
 
 				// Find the minimum checkpoint ts and resolved ts.
-				if watermark.CheckpointTs > table.Checkpoint.CheckpointTs {
+				if watermark.CheckpointTs > table.Checkpoint.CheckpointTs ||
+					(watermark.CheckpointTs == table.Checkpoint.CheckpointTs &&
+						resolvedTsOfSlowestSink > table.Checkpoint.ResolvedTs) {
 					watermark.CheckpointTs = table.Checkpoint.CheckpointTs
 					r.slowestSink = span
+					resolvedTsOfSlowestSink = table.Checkpoint.ResolvedTs
 				}
 				if watermark.ResolvedTs > table.Checkpoint.ResolvedTs {
 					watermark.ResolvedTs = table.Checkpoint.ResolvedTs
@@ -854,7 +858,7 @@ func (r *Manager) CollectMetrics() {
 			Set(float64(counter))
 	}
 
-	if table, ok := r.spans.Get(r.slowestSink); ok {
+	if table, ok := r.spans.Get(r.slowestPuller); ok {
 		if pullerCkpt, ok := table.Stats.StageCheckpoints["puller-egress"]; ok {
 			phyCkptTs := oracle.ExtractPhysical(pullerCkpt.ResolvedTs)
 			slowestTablePullerResolvedTs.WithLabelValues(cf.Namespace, cf.ID).Set(float64(phyCkptTs))

diff --git a/metrics/alertmanager/ticdc.rules.yml b/metrics/alertmanager/ticdc.rules.yml
@@ -101,7 +101,7 @@ groups:
 
   # tikv related alter rules
   - alert: tikv_cdc_min_resolved_ts_no_change_for_1m
-    expr: changes(tikv_cdc_min_resolved_ts[1m]) < 1 and ON (instance) tikv_cdc_region_resolve_status{status="resolved"} > 0
+    expr: changes(tikv_cdc_min_resolved_ts[1m]) < 1 and ON (instance) tikv_cdc_region_resolve_status{status="resolved"} > 0 and ON (instance) tikv_cdc_captured_region_total > 0
     for: 1m
     labels:
       env: ENV_LABELS_ENV