From 8a0cb8078737677228ccc51e335a62f31321cd01 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 18 Jun 2023 13:10:05 -0400 Subject: [PATCH] Remove CI endtoend test for VReplication copy throttling Improve the unit test and rely on that for testing the feature. This never really needed to have its own e2e test as it's a minor feature and pretty basic behavior. Signed-off-by: Matt Lord --- .../vreplication/vreplication_test.go | 7 ++ .../tabletserver/vstreamer/engine_test.go | 64 +++++++++++-------- test/config.json | 9 --- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 5693dfd14a7..38a1747103a 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -221,6 +221,13 @@ func TestVReplicationDDLHandling(t *testing.T) { moveTablesAction(t, "Cancel", defaultCellName, workflow, sourceKs, targetKs, table) } +// TestVreplicationCopyThrottling tests the logic that is used +// by vstreamer when starting a copy phase cycle. +// This logic today supports waiting for MySQL replication lag +// and/or InnoDB MVCC history to be below a certain threshold +// before starting the next copy phase. This test focuses on +// the innodb history list length check. +// NOTE: this is a manual test. It is not executed in the CI. func TestVreplicationCopyThrottling(t *testing.T) { workflow := "copy-throttling" cell := "zone1" diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go index e73912b6088..7324e59c3b9 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go @@ -164,8 +164,30 @@ func expectUpdateCount(t *testing.T, wantCount int64) int64 { panic("unreachable") } +// TestVStreamerWaitForMySQL tests the wait for MySQL to catch-up +// logic that is used by vstreamer when starting a copy phase cycle. +// This logic today supports waiting for MySQL replication lag +// and/or InnoDB MVCC history to be below a certain threshold before +// starting the next copy phase. func TestVStreamerWaitForMySQL(t *testing.T) { tableName := "test" + expectedWaits := int64(0) + testDB := fakesqldb.New(t) + hostres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "hostname|port", + "varchar|int64"), + "localhost|3306", + ) + thlres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "history_len", + "int64"), + "1000", + ) + sbmres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "Seconds_Behind_Master", + "int64"), + "10", + ) type fields struct { vse *Engine cp dbconfigs.Connector @@ -175,19 +197,21 @@ func TestVStreamerWaitForMySQL(t *testing.T) { maxMySQLReplLagSecs int64 } tests := []struct { - name string - fields fields - wantErr bool + name string + fields fields + shouldWait bool + wantErr bool }{ { name: "Small InnoDB MVCC impact limit", fields: fields{ vse: engine, se: engine.se, - maxInnoDBTrxHistLen: 100, + maxInnoDBTrxHistLen: 100, // Should wait on this maxMySQLReplLagSecs: 5000, }, - wantErr: true, + shouldWait: true, + wantErr: true, }, { name: "Small Repl Lag impact limit", @@ -195,9 +219,10 @@ func TestVStreamerWaitForMySQL(t *testing.T) { vse: engine, se: engine.se, maxInnoDBTrxHistLen: 10000, - maxMySQLReplLagSecs: 5, + maxMySQLReplLagSecs: 5, // Should wait on this }, - wantErr: true, + shouldWait: true, + wantErr: true, }, { name: "Large impact limits", @@ -210,25 +235,11 @@ func TestVStreamerWaitForMySQL(t *testing.T) { wantErr: false, }, } - testDB := fakesqldb.New(t) - hostres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( - "hostname|port", - "varchar|int64"), - "localhost|3306", - ) - thlres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( - "history_len", - "int64"), - "1000", - ) - sbmres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( - "Seconds_Behind_Master", - "int64"), - "10", - ) + testDB.AddQuery(hostQuery, hostres) testDB.AddQuery(trxHistoryLenQuery, thlres) testDB.AddQuery(replicaLagQuery, sbmres) + for _, tt := range tests { tt.fields.cp = testDB.ConnParams() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -247,9 +258,12 @@ func TestVStreamerWaitForMySQL(t *testing.T) { if err := uvs.vse.waitForMySQL(ctx, uvs.cp, tableName); (err != nil) != tt.wantErr { t.Errorf("vstreamer.waitForMySQL() error = %v, wantErr %v", err, tt.wantErr) } + if tt.shouldWait { + expectedWaits++ + } }) } - require.Equal(t, engine.rowStreamerWaits.Counts()["VStreamerTest.waitForMySQL"], int64(2)) - require.Equal(t, engine.vstreamerPhaseTimings.Counts()["VStreamerTest."+tableName+":waitForMySQL"], int64(2)) + require.Equal(t, engine.rowStreamerWaits.Counts()["VStreamerTest.waitForMySQL"], expectedWaits) + require.Equal(t, engine.vstreamerPhaseTimings.Counts()["VStreamerTest."+tableName+":waitForMySQL"], expectedWaits) } diff --git a/test/config.json b/test/config.json index 0bf5c2ab31a..0070720f1a0 100644 --- a/test/config.json +++ b/test/config.json @@ -1035,15 +1035,6 @@ "RetryMax": 1, "Tags": [] }, - "vreplication_copy_throttling": { - "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVreplicationCopyThrottling"], - "Command": [], - "Manual": false, - "Shard": "vreplication_basic", - "RetryMax": 1, - "Tags": [] - }, "vreplication_copy_parallel": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVreplicationCopyParallel"],