From f1d4db97e02ea235272ba2db1646624bb4807d9c Mon Sep 17 00:00:00 2001 From: lance6716 Date: Fri, 14 Jan 2022 14:25:41 +0800 Subject: [PATCH 1/2] source_worker(dm): peroidically update source metrics background --- dm/dm/worker/source_worker.go | 3 +++ dm/tests/metrics/run.sh | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/dm/dm/worker/source_worker.go b/dm/dm/worker/source_worker.go index b9125c5c1b2..ffd31441ad6 100644 --- a/dm/dm/worker/source_worker.go +++ b/dm/dm/worker/source_worker.go @@ -181,6 +181,9 @@ func (w *SourceWorker) Start() { if w.l.Core().Enabled(zap.DebugLevel) { w.l.Debug("runtime status", zap.String("status", w.GetUnitAndSourceStatusJSON("", sourceStatus))) } + + // periodically print the status and update metrics + w.Status("", sourceStatus) } } } diff --git a/dm/tests/metrics/run.sh b/dm/tests/metrics/run.sh index 9372497d489..9dbeb3b4420 100755 --- a/dm/tests/metrics/run.sh +++ b/dm/tests/metrics/run.sh @@ -20,6 +20,7 @@ function run() { inject_points=( "github.com/pingcap/tiflow/dm/syncer/BlockDDLJob=return(1)" "github.com/pingcap/tiflow/dm/syncer/ShowLagInLog=return(1)" # test lag metric >= 1 beacuse we inject BlockDDLJob(ddl) to sleep(1) + "github.com/pingcap/tiflow/dm/dm/worker/PrintStatusCheckSeconds=return(1)" ) export GO_FAILPOINTS="$(join_string \; ${inject_points[@]})" @@ -54,6 +55,11 @@ function run() { check_metric $WORKER1_PORT 'dm_worker_task_state{source_id="mysql-replica-01",task="test",worker="worker1"}' 10 1 3 check_metric $WORKER2_PORT 'dm_worker_task_state{source_id="mysql-replica-02",task="test",worker="worker2"}' 10 1 3 + # check dm_syncer_binlog_file is updated timely + run_sql_source1 "flush logs;" + check_metric $WORKER1_PORT 'dm_syncer_binlog_file{node="syncer",source_id="mysql-replica-01",task="test"}' 10 1 3 + check_metric $WORKER1_PORT 'dm_syncer_binlog_file{node="master",source_id="mysql-replica-01",task="test"}' 10 1 3 + # check ddl job lag run_sql_source1 "alter table metrics.t1 add column new_col1 int;" run_sql_source2 "alter table metrics.t2 add column new_col1 int;" From eb60b6bb5d2c49389cf6d81be5b4fd7764ce35b1 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Fri, 14 Jan 2022 14:27:51 +0800 Subject: [PATCH 2/2] fix fmt --- dm/tests/metrics/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dm/tests/metrics/run.sh b/dm/tests/metrics/run.sh index 9dbeb3b4420..9ecf992fc75 100755 --- a/dm/tests/metrics/run.sh +++ b/dm/tests/metrics/run.sh @@ -55,8 +55,8 @@ function run() { check_metric $WORKER1_PORT 'dm_worker_task_state{source_id="mysql-replica-01",task="test",worker="worker1"}' 10 1 3 check_metric $WORKER2_PORT 'dm_worker_task_state{source_id="mysql-replica-02",task="test",worker="worker2"}' 10 1 3 - # check dm_syncer_binlog_file is updated timely - run_sql_source1 "flush logs;" + # check dm_syncer_binlog_file is updated timely + run_sql_source1 "flush logs;" check_metric $WORKER1_PORT 'dm_syncer_binlog_file{node="syncer",source_id="mysql-replica-01",task="test"}' 10 1 3 check_metric $WORKER1_PORT 'dm_syncer_binlog_file{node="master",source_id="mysql-replica-01",task="test"}' 10 1 3