From a5ab3c61682515e36af1e9d5562d56422304a3d5 Mon Sep 17 00:00:00 2001 From: austenLacy Date: Thu, 27 Apr 2023 10:48:49 -0400 Subject: [PATCH 1/2] use switcher struct when switching shard reads during a reshard event Signed-off-by: austenLacy --- go/vt/wrangler/traffic_switcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/wrangler/traffic_switcher.go b/go/vt/wrangler/traffic_switcher.go index 35ee3eb1b48..af48f73e30e 100644 --- a/go/vt/wrangler/traffic_switcher.go +++ b/go/vt/wrangler/traffic_switcher.go @@ -397,7 +397,7 @@ func (wr *Wrangler) SwitchReads(ctx context.Context, targetKeyspace, workflowNam return sw.logs(), nil } wr.Logger().Infof("About to switchShardReads: %+v, %+v, %+v", cells, servedTypes, direction) - if err := ts.switchShardReads(ctx, cells, servedTypes, direction); err != nil { + if err := sw.switchShardReads(ctx, cells, servedTypes, direction); err != nil { ts.Logger().Errorf("switchShardReads failed: %v", err) return nil, err } From 99540f4ac2a328f236f1d3ae418418d86c3c9fce Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 28 Apr 2023 11:18:23 +0200 Subject: [PATCH 2/2] Create failing test for bug reported in #12992, where a TrafficSwitch dry run for reads during resharding tries to actually switch reads and fails Signed-off-by: Rohit Nayak --- .../vreplication/vreplication_test.go | 19 ++++++++++++------- .../vreplication/vreplication_test_env.go | 6 ++++++ go/test/endtoend/vreplication/vstream_test.go | 4 ++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index fafc3d91b39..5448a4a512f 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -911,7 +911,7 @@ func reshardCustomer2to4Split(t *testing.T, cells []*Cell, sourceCellOrAlias str ksName := "customer" counts := map[string]int{"zone1-600": 4, "zone1-700": 5, "zone1-800": 6, "zone1-900": 5} reshard(t, ksName, "customer", "c2c4", "-80,80-", "-40,40-80,80-c0,c0-", - 600, counts, nil, cells, sourceCellOrAlias, 1) + 600, counts, nil, nil, cells, sourceCellOrAlias, 1) waitForRowCount(t, vtgateConn, ksName, "customer", 20) query := "insert into customer (name) values('yoko')" execVtgateQuery(t, vtgateConn, ksName, query) @@ -924,7 +924,7 @@ func reshardMerchant2to3SplitMerge(t *testing.T) { ksName := merchantKeyspace counts := map[string]int{"zone1-1600": 0, "zone1-1700": 2, "zone1-1800": 0} reshard(t, ksName, "merchant", "m2m3", "-80,80-", "-40,40-c0,c0-", - 1600, counts, dryRunResultsSwitchWritesM2m3, nil, "", 1) + 1600, counts, dryRunResultsSwitchReadM2m3, dryRunResultsSwitchWritesM2m3, nil, "", 1) waitForRowCount(t, vtgateConn, ksName, "merchant", 2) query := "insert into merchant (mname, category) values('amazon', 'electronics')" execVtgateQuery(t, vtgateConn, ksName, query) @@ -971,7 +971,7 @@ func reshardMerchant3to1Merge(t *testing.T) { ksName := merchantKeyspace counts := map[string]int{"zone1-2000": 3} reshard(t, ksName, "merchant", "m3m1", "-40,40-c0,c0-", "0", - 2000, counts, nil, nil, "", 1) + 2000, counts, nil, nil, nil, "", 1) waitForRowCount(t, vtgateConn, ksName, "merchant", 3) query := "insert into merchant (mname, category) values('flipkart', 'electronics')" execVtgateQuery(t, vtgateConn, ksName, query) @@ -984,7 +984,7 @@ func reshardCustomer3to2SplitMerge(t *testing.T) { // -40,40-80,80-c0 => merge/s ksName := "customer" counts := map[string]int{"zone1-1000": 8, "zone1-1100": 8, "zone1-1200": 5} reshard(t, ksName, "customer", "c4c3", "-40,40-80,80-c0", "-60,60-c0", - 1000, counts, nil, nil, "", 1) + 1000, counts, nil, nil, nil, "", 1) }) } @@ -993,12 +993,12 @@ func reshardCustomer3to1Merge(t *testing.T) { // to unsharded ksName := "customer" counts := map[string]int{"zone1-1500": 21} reshard(t, ksName, "customer", "c3c1", "-60,60-c0,c0-", "0", - 1500, counts, nil, nil, "", 3) + 1500, counts, nil, nil, nil, "", 3) }) } func reshard(t *testing.T, ksName string, tableName string, workflow string, sourceShards string, targetShards string, - tabletIDBase int, counts map[string]int, dryRunResultSwitchWrites []string, cells []*Cell, sourceCellOrAlias string, + tabletIDBase int, counts map[string]int, dryRunResultSwitchReads, dryRunResultSwitchWrites []string, cells []*Cell, sourceCellOrAlias string, autoIncrementStep int) { t.Run("reshard", func(t *testing.T) { if cells == nil { @@ -1040,6 +1040,9 @@ func reshard(t *testing.T, ksName string, tableName string, workflow string, sou } } vdiff1(t, ksWorkflow, "") + if dryRunResultSwitchReads != nil { + switchReadsDryRun(t, workflowType, allCellNames, ksWorkflow, dryRunResultSwitchReads) + } switchReads(t, workflowType, allCellNames, ksWorkflow, false) if dryRunResultSwitchWrites != nil { switchWritesDryRun(t, workflowType, ksWorkflow, dryRunResultSwitchWrites) @@ -1397,7 +1400,9 @@ func switchReadsDryRun(t *testing.T, workflowType, cells, ksWorkflow string, dry output, err := vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--tablet_types=rdonly,replica", "--dry_run", "SwitchTraffic", ksWorkflow) require.NoError(t, err, fmt.Sprintf("Switching Reads DryRun Error: %s: %s", err, output)) - validateDryRunResults(t, output, dryRunResults) + if dryRunResults != nil { + validateDryRunResults(t, output, dryRunResults) + } } func switchReads(t *testing.T, workflowType, cells, ksWorkflow string, reverse bool) { diff --git a/go/test/endtoend/vreplication/vreplication_test_env.go b/go/test/endtoend/vreplication/vreplication_test_env.go index 9e88dc99460..2d98fcb0029 100644 --- a/go/test/endtoend/vreplication/vreplication_test_env.go +++ b/go/test/endtoend/vreplication/vreplication_test_env.go @@ -85,3 +85,9 @@ var dryRunResultsSwitchWritesM2m3 = []string{ " Keyspace merchant-type, Shard c0-, Tablet 1800, Workflow m2m3, DbName vt_merchant-type", "Unlock keyspace merchant-type", } + +var dryRunResultsSwitchReadM2m3 = []string{ + "Lock keyspace merchant-type", + "Switch reads from keyspace merchant-type to keyspace merchant-type for shards -80,80- to shards -40,40-c0,c0-", + "Unlock keyspace merchant-type", +} diff --git a/go/test/endtoend/vreplication/vstream_test.go b/go/test/endtoend/vreplication/vstream_test.go index aec244a0389..63c6655cf5c 100644 --- a/go/test/endtoend/vreplication/vstream_test.go +++ b/go/test/endtoend/vreplication/vstream_test.go @@ -372,7 +372,7 @@ func testVStreamStopOnReshardFlag(t *testing.T, stopOnReshard bool, baseTabletID switch tickCount { case 1: reshard(t, "sharded", "customer", "vstreamStopOnReshard", "-80,80-", - "-40,40-", baseTabletID+400, nil, nil, nil, defaultCellName, 1) + "-40,40-", baseTabletID+400, nil, nil, nil, nil, defaultCellName, 1) case 60: done = true } @@ -502,7 +502,7 @@ func testVStreamCopyMultiKeyspaceReshard(t *testing.T, baseTabletID int) numEven tickCount++ switch tickCount { case 1: - reshard(t, "sharded", "customer", "vstreamCopyMultiKeyspaceReshard", "-80,80-", "-40,40-", baseTabletID+400, nil, nil, nil, defaultCellName, 1) + reshard(t, "sharded", "customer", "vstreamCopyMultiKeyspaceReshard", "-80,80-", "-40,40-", baseTabletID+400, nil, nil, nil, nil, defaultCellName, 1) reshardDone = true case 60: done = true