From 49b7cc6a3c0efe3ead3f67be5cea8e682ce32960 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Tue, 8 Aug 2023 17:21:25 +0800 Subject: [PATCH] calibrate: shorten the minimum calibrate duration (#44972) ref pingcap/tidb#43212 --- .../calibrateresource/calibrate_resource.go | 9 +- .../calibrate_resource_test.go | 92 ++++++++++--------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/executor/internal/calibrateresource/calibrate_resource.go b/executor/internal/calibrateresource/calibrate_resource.go index 2bec8e0351af1..547316d49be5d 100644 --- a/executor/internal/calibrateresource/calibrate_resource.go +++ b/executor/internal/calibrateresource/calibrate_resource.go @@ -105,14 +105,12 @@ const ( // lowUsageThreshold is the threshold used to determine whether the CPU is too low. // When the CPU utilization of tikv or tidb is lower than lowUsageThreshold, but neither is higher than valuableUsageThreshold, the sampling point is unavailable lowUsageThreshold = 0.1 - // calibration is performed only when the available time point exceeds the percentOfPass - percentOfPass = 0.9 // For quotas computed at each point in time, the maximum and minimum portions are discarded, and discardRate is the percentage discarded discardRate = 0.1 // duration Indicates the supported calibration duration maxDuration = time.Hour * 24 - minDuration = time.Minute * 10 + minDuration = time.Minute ) // Executor is used as executor of calibrate resource. @@ -334,10 +332,7 @@ func (e *Executor) dynamicCalibrate(ctx context.Context, req *chunk.Chunk, exec tidbCPUs.next() tikvCPUs.next() } - if len(quotas) < 5 { - return errLowUsage - } - if float64(len(quotas))/float64(len(quotas)+lowCount) <= percentOfPass { + if len(quotas) < 2 { return errLowUsage } sort.Slice(quotas, func(i, j int) bool { diff --git a/executor/internal/calibrateresource/calibrate_resource_test.go b/executor/internal/calibrateresource/calibrate_resource_test.go index 58af3d263800d..b0aaaa9c7b607 100644 --- a/executor/internal/calibrateresource/calibrate_resource_test.go +++ b/executor/internal/calibrateresource/calibrate_resource_test.go @@ -134,6 +134,7 @@ func TestCalibrateResource(t *testing.T) { tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE").Check(testkit.Rows("38760")) ru1 := [][]types.Datum{ + types.MakeDatums(datetimeBeforeNow(20*time.Minute+40*time.Second), 2250.0), types.MakeDatums(datetimeBeforeNow(20*time.Minute+10*time.Second), 2200.0), types.MakeDatums(datetimeBeforeNow(10*time.Minute+10*time.Second), 2200.0), types.MakeDatums(datetimeBeforeNow(9*time.Minute+10*time.Second), 2100.0), @@ -149,6 +150,7 @@ func TestCalibrateResource(t *testing.T) { } mockData["resource_manager_resource_unit"] = ru1 cpu1 := [][]types.Datum{ + types.MakeDatums(datetimeBeforeNow(20*time.Minute+40*time.Second), "tidb-0", "tidb", 1.234), types.MakeDatums(datetimeBeforeNow(20*time.Minute+10*time.Second), "tidb-0", "tidb", 1.212), types.MakeDatums(datetimeBeforeNow(10*time.Minute+10*time.Second), "tidb-0", "tidb", 1.212), types.MakeDatums(datetimeBeforeNow(9*time.Minute+10*time.Second), "tidb-0", "tidb", 1.233), @@ -161,6 +163,7 @@ func TestCalibrateResource(t *testing.T) { types.MakeDatums(datetimeBeforeNow(2*time.Minute+10*time.Second), "tidb-0", "tidb", 1.219), types.MakeDatums(datetimeBeforeNow(1*time.Minute+10*time.Second), "tidb-0", "tidb", 1.220), types.MakeDatums(datetimeBeforeNow(10*time.Second), "tidb-0", "tidb", 1.221), + types.MakeDatums(datetimeBeforeNow(20*time.Minute+40*time.Second), "tikv-1", "tikv", 2.219), types.MakeDatums(datetimeBeforeNow(20*time.Minute+10*time.Second), "tikv-1", "tikv", 2.212), types.MakeDatums(datetimeBeforeNow(10*time.Minute+10*time.Second), "tikv-1", "tikv", 2.212), types.MakeDatums(datetimeBeforeNow(9*time.Minute+10*time.Second), "tikv-1", "tikv", 2.233), @@ -173,6 +176,7 @@ func TestCalibrateResource(t *testing.T) { types.MakeDatums(datetimeBeforeNow(2*time.Minute+10*time.Second), "tikv-1", "tikv", 2.219), types.MakeDatums(datetimeBeforeNow(1*time.Minute+10*time.Second), "tikv-1", "tikv", 2.220), types.MakeDatums(datetimeBeforeNow(10*time.Second), "tikv-1", "tikv", 2.281), + types.MakeDatums(datetimeBeforeNow(20*time.Minute+40*time.Second), "tikv-0", "tikv", 2.280), types.MakeDatums(datetimeBeforeNow(20*time.Minute+10*time.Second), "tikv-0", "tikv", 2.282), types.MakeDatums(datetimeBeforeNow(10*time.Minute+10*time.Second), "tikv-0", "tikv", 2.282), types.MakeDatums(datetimeBeforeNow(9*time.Minute+10*time.Second), "tikv-0", "tikv", 2.283), @@ -185,6 +189,7 @@ func TestCalibrateResource(t *testing.T) { types.MakeDatums(datetimeBeforeNow(2*time.Minute+10*time.Second), "tikv-0", "tikv", 2.289), types.MakeDatums(datetimeBeforeNow(1*time.Minute+10*time.Second), "tikv-0", "tikv", 2.280), types.MakeDatums(datetimeBeforeNow(10*time.Second), "tikv-0", "tikv", 2.281), + types.MakeDatums(datetimeBeforeNow(20*time.Minute+40*time.Second), "tikv-2", "tikv", 2.281), types.MakeDatums(datetimeBeforeNow(20*time.Minute+10*time.Second), "tikv-2", "tikv", 2.112), types.MakeDatums(datetimeBeforeNow(10*time.Minute+10*time.Second), "tikv-2", "tikv", 2.112), types.MakeDatums(datetimeBeforeNow(9*time.Minute+10*time.Second), "tikv-2", "tikv", 2.133), @@ -208,10 +213,21 @@ func TestCalibrateResource(t *testing.T) { tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 11 minute DURATION interval 11 minute").Check(testkit.Rows("8161")) tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 11 minute START_TIME now() - interval 11 minute").Check(testkit.Rows("8161")) tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE END_TIME now() DURATION interval 11 minute").Check(testkit.Rows("8161")) - tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 21 minute END_TIME now() - interval 1 minute").Check(testkit.Rows("8160")) - tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION '20m' START_TIME now() - interval 21 minute").Check(testkit.Rows("8160")) - tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION '20m' END_TIME now() - interval 1 minute").Check(testkit.Rows("8160")) - tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 20 minute START_TIME now() - interval 21 minute").Check(testkit.Rows("8160")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 21 minute END_TIME now() - interval 1 minute").Check(testkit.Rows("8141")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION '20m' START_TIME now() - interval 21 minute").Check(testkit.Rows("8141")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 20 minute START_TIME now() - interval 21 minute").Check(testkit.Rows("8141")) + + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 21 minute END_TIME now() - interval 20 minute").Check(testkit.Rows("7978")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 4 minute").Check(testkit.Rows("8297")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 4 minute").Check(testkit.Rows("8297")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 4 minute END_TIME now()").Check(testkit.Rows("8297")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 8 minute").Check(testkit.Rows("8223")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 8 minute").Check(testkit.Rows("8223")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 8 minute END_TIME now() - interval 4 minute").Check(testkit.Rows("8147")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - interval 8 minute DURATION interval 4 minute").Check(testkit.Rows("8147")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 4 minute START_TIME now() - interval 8 minute").Check(testkit.Rows("8147")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE DURATION interval 4 minute END_TIME now() - interval 4 minute").Check(testkit.Rows("8147")) + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE END_TIME date_sub(now(), interval 4 minute ) DURATION '4m'").Check(testkit.Rows("8147")) // construct data for dynamic calibrate ru1 = [][]types.Datum{ @@ -542,13 +558,9 @@ func TestCalibrateResource(t *testing.T) { err = rs.Next(ctx, rs.NewChunk(nil)) require.ErrorContains(t, err, "the duration of calibration is too long") - rs, err = tk.Exec("CALIBRATE RESOURCE START_TIME '2020-02-12 10:35:00' DURATION '5m'") - require.NoError(t, err) - require.NotNil(t, rs) - err = rs.Next(ctx, rs.NewChunk(nil)) - require.ErrorContains(t, err, "the duration of calibration is too short") + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME '2020-02-12 10:35:00' DURATION '1m'").Check(testkit.Rows("5337")) - rs, err = tk.Exec("CALIBRATE RESOURCE START_TIME '2020-02-12 10:35:00' END_TIME '2020-02-12 10:43:00'") + rs, err = tk.Exec("CALIBRATE RESOURCE START_TIME '2020-02-12 10:35:00' END_TIME '2020-02-12 10:35:40'") require.NoError(t, err) require.NotNil(t, rs) err = rs.Next(ctx, rs.NewChunk(nil)) @@ -567,47 +579,47 @@ func TestCalibrateResource(t *testing.T) { require.ErrorContains(t, err, "the duration of calibration is too long") mockData["process_cpu_usage"] = [][]types.Datum{ - types.MakeDatums(datetime("2020-02-12 10:35:00"), "tidb-0", "tidb", 3.212), - types.MakeDatums(datetime("2020-02-12 10:36:00"), "tidb-0", "tidb", 3.233), - types.MakeDatums(datetime("2020-02-12 10:37:00"), "tidb-0", "tidb", 3.234), + types.MakeDatums(datetime("2020-02-12 10:35:00"), "tidb-0", "tidb", 0.212), + types.MakeDatums(datetime("2020-02-12 10:36:00"), "tidb-0", "tidb", 0.233), + types.MakeDatums(datetime("2020-02-12 10:37:00"), "tidb-0", "tidb", 0.234), types.MakeDatums(datetime("2020-02-12 10:38:00"), "tidb-0", "tidb", 3.213), - types.MakeDatums(datetime("2020-02-12 10:39:00"), "tidb-0", "tidb", 3.209), - types.MakeDatums(datetime("2020-02-12 10:40:00"), "tidb-0", "tidb", 3.213), - types.MakeDatums(datetime("2020-02-12 10:41:00"), "tidb-0", "tidb", 3.236), - types.MakeDatums(datetime("2020-02-12 10:42:00"), "tidb-0", "tidb", 3.228), + types.MakeDatums(datetime("2020-02-12 10:39:00"), "tidb-0", "tidb", 0.209), + types.MakeDatums(datetime("2020-02-12 10:40:00"), "tidb-0", "tidb", 0.213), + types.MakeDatums(datetime("2020-02-12 10:41:00"), "tidb-0", "tidb", 0.236), + types.MakeDatums(datetime("2020-02-12 10:42:00"), "tidb-0", "tidb", 0.228), types.MakeDatums(datetime("2020-02-12 10:43:00"), "tidb-0", "tidb", 0.219), types.MakeDatums(datetime("2020-02-12 10:44:00"), "tidb-0", "tidb", 0.220), types.MakeDatums(datetime("2020-02-12 10:45:00"), "tidb-0", "tidb", 0.221), types.MakeDatums(datetime("2020-02-12 10:35:00"), "tikv-1", "tikv", 2.212), - types.MakeDatums(datetime("2020-02-12 10:36:00"), "tikv-1", "tikv", 2.233), - types.MakeDatums(datetime("2020-02-12 10:37:00"), "tikv-1", "tikv", 2.234), - types.MakeDatums(datetime("2020-02-12 10:38:00"), "tikv-1", "tikv", 2.213), - types.MakeDatums(datetime("2020-02-12 10:39:00"), "tikv-1", "tikv", 2.209), + types.MakeDatums(datetime("2020-02-12 10:36:00"), "tikv-1", "tikv", 0.233), + types.MakeDatums(datetime("2020-02-12 10:37:00"), "tikv-1", "tikv", 0.234), + types.MakeDatums(datetime("2020-02-12 10:38:00"), "tikv-1", "tikv", 0.213), + types.MakeDatums(datetime("2020-02-12 10:39:00"), "tikv-1", "tikv", 0.209), types.MakeDatums(datetime("2020-02-12 10:40:00"), "tikv-1", "tikv", 2.213), - types.MakeDatums(datetime("2020-02-12 10:41:00"), "tikv-1", "tikv", 2.236), - types.MakeDatums(datetime("2020-02-12 10:42:00"), "tikv-1", "tikv", 2.228), + types.MakeDatums(datetime("2020-02-12 10:41:00"), "tikv-1", "tikv", 0.236), + types.MakeDatums(datetime("2020-02-12 10:42:00"), "tikv-1", "tikv", 0.228), types.MakeDatums(datetime("2020-02-12 10:43:00"), "tikv-1", "tikv", 0.219), types.MakeDatums(datetime("2020-02-12 10:44:00"), "tikv-1", "tikv", 0.220), types.MakeDatums(datetime("2020-02-12 10:45:00"), "tikv-1", "tikv", 0.281), - types.MakeDatums(datetime("2020-02-12 10:35:00"), "tikv-0", "tikv", 2.282), - types.MakeDatums(datetime("2020-02-12 10:36:00"), "tikv-0", "tikv", 2.283), - types.MakeDatums(datetime("2020-02-12 10:37:00"), "tikv-0", "tikv", 2.284), + types.MakeDatums(datetime("2020-02-12 10:35:00"), "tikv-0", "tikv", 0.282), + types.MakeDatums(datetime("2020-02-12 10:36:00"), "tikv-0", "tikv", 0.283), + types.MakeDatums(datetime("2020-02-12 10:37:00"), "tikv-0", "tikv", 0.284), types.MakeDatums(datetime("2020-02-12 10:38:00"), "tikv-0", "tikv", 2.283), - types.MakeDatums(datetime("2020-02-12 10:39:00"), "tikv-0", "tikv", 2.289), - types.MakeDatums(datetime("2020-02-12 10:40:00"), "tikv-0", "tikv", 2.283), - types.MakeDatums(datetime("2020-02-12 10:41:00"), "tikv-0", "tikv", 2.286), - types.MakeDatums(datetime("2020-02-12 10:42:00"), "tikv-0", "tikv", 2.288), + types.MakeDatums(datetime("2020-02-12 10:39:00"), "tikv-0", "tikv", 0.289), + types.MakeDatums(datetime("2020-02-12 10:40:00"), "tikv-0", "tikv", 0.283), + types.MakeDatums(datetime("2020-02-12 10:41:00"), "tikv-0", "tikv", 0.286), + types.MakeDatums(datetime("2020-02-12 10:42:00"), "tikv-0", "tikv", 0.288), types.MakeDatums(datetime("2020-02-12 10:43:00"), "tikv-0", "tikv", 0.289), types.MakeDatums(datetime("2020-02-12 10:44:00"), "tikv-0", "tikv", 0.280), types.MakeDatums(datetime("2020-02-12 10:45:00"), "tikv-0", "tikv", 0.281), types.MakeDatums(datetime("2020-02-12 10:35:00"), "tikv-2", "tikv", 2.112), - types.MakeDatums(datetime("2020-02-12 10:36:00"), "tikv-2", "tikv", 2.133), - types.MakeDatums(datetime("2020-02-12 10:37:00"), "tikv-2", "tikv", 2.134), - types.MakeDatums(datetime("2020-02-12 10:38:00"), "tikv-2", "tikv", 2.113), - types.MakeDatums(datetime("2020-02-12 10:39:00"), "tikv-2", "tikv", 2.109), - types.MakeDatums(datetime("2020-02-12 10:40:00"), "tikv-2", "tikv", 2.113), - types.MakeDatums(datetime("2020-02-12 10:41:00"), "tikv-2", "tikv", 2.136), - types.MakeDatums(datetime("2020-02-12 10:42:00"), "tikv-2", "tikv", 2.128), + types.MakeDatums(datetime("2020-02-12 10:36:00"), "tikv-2", "tikv", 0.133), + types.MakeDatums(datetime("2020-02-12 10:37:00"), "tikv-2", "tikv", 0.134), + types.MakeDatums(datetime("2020-02-12 10:38:00"), "tikv-2", "tikv", 0.113), + types.MakeDatums(datetime("2020-02-12 10:39:00"), "tikv-2", "tikv", 0.109), + types.MakeDatums(datetime("2020-02-12 10:40:00"), "tikv-2", "tikv", 0.113), + types.MakeDatums(datetime("2020-02-12 10:41:00"), "tikv-2", "tikv", 0.136), + types.MakeDatums(datetime("2020-02-12 10:42:00"), "tikv-2", "tikv", 0.128), types.MakeDatums(datetime("2020-02-12 10:43:00"), "tikv-2", "tikv", 0.119), types.MakeDatums(datetime("2020-02-12 10:44:00"), "tikv-2", "tikv", 0.120), types.MakeDatums(datetime("2020-02-12 10:45:00"), "tikv-2", "tikv", 0.281), @@ -637,11 +649,7 @@ func TestCalibrateResource(t *testing.T) { types.MakeDatums(datetime("2020-02-12 10:37:00"), "tikv-2", "tikv", 2.134), types.MakeDatums(datetime("2020-02-12 10:38:00"), "tikv-2", "tikv", 2.113), } - rs, err = tk.Exec("CALIBRATE RESOURCE START_TIME '2020-02-12 10:35:00' END_TIME '2020-02-12 10:45:00'") - require.NoError(t, err) - require.NotNil(t, rs) - err = rs.Next(ctx, rs.NewChunk(nil)) - require.ErrorContains(t, err, "The workload in selected time window is too low") + tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME '2020-02-12 10:35:00' END_TIME '2020-02-12 10:45:00'").Check(testkit.Rows("5492")) delete(mockData, "process_cpu_usage") rs, err = tk.Exec("CALIBRATE RESOURCE START_TIME '2020-02-12 10:35:00' END_TIME '2020-02-12 10:45:00'")