Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/pingcap/tidb into disttas…
Browse files Browse the repository at this point in the history 
…k-rc/scheduler-slot-manager
  • Loading branch information
@okJiang
okJiang committed Dec 19, 2023
2 parents 6392faf + 0719329 commit 7d9f45e
Show file tree
Hide file tree
Showing 295 changed files with 13,119 additions and 11,725 deletions.
72 changes: 36 additions & 36 deletions DEPS.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5784,13 +5784,13 @@ def go_deps():
name = "com_github_pingcap_errors",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/errors",
sha256 = "4dadf9dc507b4187a70b78e49d572bc0e8f89a7b4a8974d6a978f72620526996",
strip_prefix = "github.com/pingcap/[email protected].20221009092201-b66cddb77c32",
sha256 = "b4db3d3c222d9039c84baacbbd9c46aa0346f3f04d2577a77475a64ecfefebf9",
strip_prefix = "github.com/pingcap/[email protected].20231212100244-799fae176cfb",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20221009092201-b66cddb77c32.zip",
"http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20221009092201-b66cddb77c32.zip",
"https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20221009092201-b66cddb77c32.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20221009092201-b66cddb77c32.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
"http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
"https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
],
)
go_repository(
Expand Down Expand Up @@ -7006,26 +7006,26 @@ def go_deps():
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
sha256 = "c6c5bfe8a84d0b680266035d1ac87af6beca4cc552693d6753d4a9f18610c30d",
strip_prefix = "github.com/tikv/client-go/[email protected].20231201024404-0ff16620f6c0",
sha256 = "04da7d520727a9140c0d472c0f0a054837aae1da3fa49101c0f52279c7d78094",
strip_prefix = "github.com/tikv/client-go/[email protected].20231204074048-e80e9ca1fe66",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231204074048-e80e9ca1fe66.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231204074048-e80e9ca1fe66.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231204074048-e80e9ca1fe66.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231204074048-e80e9ca1fe66.zip",
],
)
go_repository(
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
sha256 = "a6dcccfd436232d847d2ba2669d67d9708d7bc97f7571a4e57da18fdefeed852",
strip_prefix = "github.com/tikv/pd/[email protected]20231130081618-862eee18738e",
sha256 = "5b31b38e151e03117ef9878c2dbac2b1f22c890e72ebb70935795ac5682c77c1",
strip_prefix = "github.com/tikv/pd/[email protected]20231213112719-f51f9134558e",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231213112719-f51f9134558e.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231213112719-f51f9134558e.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231213112719-f51f9134558e.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231213112719-f51f9134558e.zip",
],
)
go_repository(
Expand Down Expand Up @@ -9177,26 +9177,26 @@ def go_deps():
name = "com_sourcegraph_sourcegraph_appdash",
build_file_proto_mode = "disable_global",
importpath = "sourcegraph.com/sourcegraph/appdash",
sha256 = "bd2492d9db05362c2fecd0b3d0f6002c89a6d90d678fb93b4158298ab883736f",
strip_prefix = "sourcegraph.com/sourcegraph/[email protected]",
sha256 = "c46b442fa40d2af48e08064f4c16ae3712953a9988cd0f7588fcf5e4fc7a2fed",
strip_prefix = "github.com/sourcegraph/[email protected]",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/sourcegraph.com/sourcegraph/appdash/com_sourcegraph_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
"http://ats.apps.svc/gomod/sourcegraph.com/sourcegraph/appdash/com_sourcegraph_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
"https://cache.hawkingrei.com/gomod/sourcegraph.com/sourcegraph/appdash/com_sourcegraph_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/sourcegraph.com/sourcegraph/appdash/com_sourcegraph_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/sourcegraph/appdash/com_github_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
"http://ats.apps.svc/gomod/github.com/sourcegraph/appdash/com_github_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
"https://cache.hawkingrei.com/gomod/github.com/sourcegraph/appdash/com_github_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/sourcegraph/appdash/com_github_sourcegraph_appdash-v0.0.0-20190731080439-ebfcffb1b5c0.zip",
],
)
go_repository(
name = "com_sourcegraph_sourcegraph_appdash_data",
build_file_proto_mode = "disable_global",
importpath = "sourcegraph.com/sourcegraph/appdash-data",
sha256 = "382adefecd62bb79172e2552bcfb7d45f47122f9bd22259b0566b26fb2627b87",
strip_prefix = "sourcegraph.com/sourcegraph/[email protected]",
sha256 = "59b71fa8cdb0fe2b1c02739ccf2daeaf28f2e22c4b178cdc8e1b902ad1022bc0",
strip_prefix = "github.com/sourcegraph/[email protected]",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/sourcegraph.com/sourcegraph/appdash-data/com_sourcegraph_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
"http://ats.apps.svc/gomod/sourcegraph.com/sourcegraph/appdash-data/com_sourcegraph_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
"https://cache.hawkingrei.com/gomod/sourcegraph.com/sourcegraph/appdash-data/com_sourcegraph_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/sourcegraph.com/sourcegraph/appdash-data/com_sourcegraph_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/sourcegraph/appdash-data/com_github_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
"http://ats.apps.svc/gomod/github.com/sourcegraph/appdash-data/com_github_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
"https://cache.hawkingrei.com/gomod/github.com/sourcegraph/appdash-data/com_github_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/sourcegraph/appdash-data/com_github_sourcegraph_appdash_data-v0.0.0-20151005221446-73f23eafcf67.zip",
],
)
go_repository(
Expand Down Expand Up @@ -10260,13 +10260,13 @@ def go_deps():
name = "org_golang_x_tools",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/tools",
sha256 = "83320ec39cdf1ace98655928e91391122f2b8b294aeedf3bf7895ad2144cb28d",
strip_prefix = "golang.org/x/[email protected].0",
sha256 = "b3f3c12d493747930eee9054fdd7e35aa36fe2c6d4b2aac2e0c29801e9fc9516",
strip_prefix = "golang.org/x/[email protected].1",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.0.zip",
"http://ats.apps.svc/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.0.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.0.zip",
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.1.zip",
"http://ats.apps.svc/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.1.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.1.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/tools/org_golang_x_tools-v0.16.1.zip",
],
)
go_repository(
Expand Down
3 changes: 2 additions & 1 deletion br/pkg/backup/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ go_test(
embed = [":backup"],
flaky = True,
race = "on",
shard_count = 9,
shard_count = 10,
deps = [
"//br/pkg/conn",
"//br/pkg/gluetidb",
Expand All @@ -89,6 +89,7 @@ go_test(
"//pkg/util/codec",
"//pkg/util/table-filter",
"@com_github_golang_protobuf//proto",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/encryptionpb",
"@com_github_pingcap_kvproto//pkg/errorpb",
Expand Down
84 changes: 67 additions & 17 deletions br/pkg/backup/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1011,23 +1011,74 @@ func (bc *Client) BackupRange(
return nil
}

func (bc *Client) findTargetPeer(ctx context.Context, key []byte, isRawKv bool, targetStoreIds map[uint64]struct{}) (*metapb.Peer, error) {
func (bc *Client) FindTargetPeer(ctx context.Context, key []byte, isRawKv bool, targetStoreIds map[uint64]struct{}) (*metapb.Peer, error) {
// Keys are saved in encoded format in TiKV, so the key must be encoded
// in order to find the correct region.
var leader *metapb.Peer
key = codec.EncodeBytesExt([]byte{}, key, isRawKv)
for i := 0; i < 5; i++ {
// better backoff.
state := utils.InitialRetryState(60, 100*time.Millisecond, 2*time.Second)
failpoint.Inject("retry-state-on-find-target-peer", func(v failpoint.Value) {
logutil.CL(ctx).Info("reset state for FindTargetPeer")
state = utils.InitialRetryState(v.(int), 100*time.Millisecond, 100*time.Millisecond)
})
err := utils.WithRetry(ctx, func() error {
region, err := bc.mgr.GetPDClient().GetRegion(ctx, key)
failpoint.Inject("return-region-on-find-target-peer", func(v failpoint.Value) {
switch v.(string) {
case "nil":
{
region = nil
}
case "hasLeader":
{
region = &pd.Region{
Leader: &metapb.Peer{
Id: 42,
},
}
}
case "hasPeer":
{
region = &pd.Region{
Meta: &metapb.Region{
Peers: []*metapb.Peer{
{
Id: 43,
StoreId: 42,
},
},
},
}
}

case "noLeader":
{
region = &pd.Region{
Leader: nil,
}
}
case "noPeer":
{
{
region = &pd.Region{
Meta: &metapb.Region{
Peers: nil,
},
}
}
}
}
})
if err != nil || region == nil {
logutil.CL(ctx).Error("find region failed", zap.Error(err), zap.Reflect("region", region))
time.Sleep(time.Millisecond * time.Duration(100*i))
continue
return errors.Annotate(berrors.ErrPDLeaderNotFound, "cannot find region from pd client")
}
if len(targetStoreIds) == 0 {
if region.Leader != nil {
logutil.CL(ctx).Info("find leader",
zap.Reflect("Leader", region.Leader), logutil.Key("key", key))
return region.Leader, nil
leader = region.Leader
return nil
}
} else {
candidates := make([]*metapb.Peer, 0, len(region.Meta.Peers))
Expand All @@ -1040,19 +1091,18 @@ func (bc *Client) findTargetPeer(ctx context.Context, key []byte, isRawKv bool,
peer := candidates[rand.Intn(len(candidates))]
logutil.CL(ctx).Info("find target peer for backup",
zap.Reflect("Peer", peer), logutil.Key("key", key))
return peer, nil
leader = peer
return nil
}
}

logutil.CL(ctx).Warn("fail to find a target peer", logutil.Key("key", key))
time.Sleep(time.Millisecond * time.Duration(1000*i))
continue
}
logutil.CL(ctx).Error("can not find a valid target peer", logutil.Key("key", key))
if len(targetStoreIds) == 0 {
return nil, errors.Annotatef(berrors.ErrBackupNoLeader, "can not find a valid leader for key %s", key)
return errors.Annotate(berrors.ErrPDLeaderNotFound, "cannot find leader or candidate from pd client")
}, &state)
if err != nil {
logutil.CL(ctx).Error("can not find a valid target peer after retry", logutil.Key("key", key))
return nil, err
}
return nil, errors.Errorf("can not find a valid target peer for key %s", key)
// leader cannot be nil if err is nil
return leader, nil
}

func (bc *Client) fineGrainedBackup(
Expand Down Expand Up @@ -1226,7 +1276,7 @@ func (bc *Client) handleFineGrained(
targetStoreIds map[uint64]struct{},
respCh chan<- *backuppb.BackupResponse,
) (int, error) {
targetPeer, pderr := bc.findTargetPeer(ctx, req.StartKey, req.IsRawKv, targetStoreIds)
targetPeer, pderr := bc.FindTargetPeer(ctx, req.StartKey, req.IsRawKv, targetStoreIds)
if pderr != nil {
return 0, errors.Trace(pderr)
}
Expand Down
65 changes: 64 additions & 1 deletion br/pkg/backup/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"time"

"github.com/golang/protobuf/proto"
"github.com/pingcap/failpoint"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/encryptionpb"
"github.com/pingcap/kvproto/pkg/errorpb"
Expand Down Expand Up @@ -41,6 +42,7 @@ type testBackup struct {
cancel context.CancelFunc

mockPDClient pd.Client
mockCluster *testutils.MockCluster
mockGlue *gluetidb.MockGlue
backupClient *backup.Client

Expand All @@ -49,11 +51,12 @@ type testBackup struct {
}

func createBackupSuite(t *testing.T) *testBackup {
tikvClient, _, pdClient, err := testutils.NewMockTiKV("", nil)
tikvClient, mockCluster, pdClient, err := testutils.NewMockTiKV("", nil)
require.NoError(t, err)
s := new(testBackup)
s.mockGlue = &gluetidb.MockGlue{}
s.mockPDClient = pdClient
s.mockCluster = mockCluster
s.ctx, s.cancel = context.WithCancel(context.Background())
mockMgr := &conn.Mgr{PdController: &pdutil.PdController{}}
mockMgr.SetPDClient(s.mockPDClient)
Expand Down Expand Up @@ -334,3 +337,63 @@ func TestCheckBackupIsLocked(t *testing.T) {
require.Error(t, err)
require.Regexp(t, "backup lock file and sst file exist in(.+)", err.Error())
}

func TestFindTargetPeer(t *testing.T) {
s := createBackupSuite(t)

ctx := context.Background()
testutils.BootstrapWithMultiRegions(s.mockCluster, []byte("g"), []byte("n"), []byte("t"))

leader1, err := s.backupClient.FindTargetPeer(ctx, []byte("a"), false, nil)
require.NoError(t, err)

leader2, err := s.backupClient.FindTargetPeer(ctx, []byte("b"), false, nil)
require.NoError(t, err)

// check passed keys on same region
require.Equal(t, leader1.GetId(), leader2.GetId())

failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer", "return(2)")
failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer", "1*return(\"nil\")->1*return(\"hasLeader\")")

leader, err := s.backupClient.FindTargetPeer(ctx, []byte("m"), false, nil)
require.NoError(t, err)
// check passed keys on find leader after retry
require.Equal(t, 42, int(leader.GetId()))

failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer")
failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer")

failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer", "return(2)")
failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer", "return(\"noLeader\")")

leader, err = s.backupClient.FindTargetPeer(ctx, []byte("m"), false, nil)
// check passed keys with error on find leader after retry
require.ErrorContains(t, err, "cannot find leader")

failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer")
failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer")

failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer", "return(2)")
failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer", "1*return(\"nil\")->1*return(\"hasPeer\")")

storeIDMap := make(map[uint64]struct{})
storeIDMap[42] = struct{}{}
leader, err = s.backupClient.FindTargetPeer(ctx, []byte("m"), false, storeIDMap)
require.NoError(t, err)
// check passed keys with target peer
require.Equal(t, 43, int(leader.GetId()))

failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer")
failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer")

failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer", "return(2)")
failpoint.Enable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer", "1*return(\"nil\")->1*return(\"noPeer\")")

leader, err = s.backupClient.FindTargetPeer(ctx, []byte("m"), false, storeIDMap)
// check passed keys with error and cannot find target peer
require.ErrorContains(t, err, "cannot find leader")

failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/retry-state-on-find-target-peer")
failpoint.Disable("github.com/pingcap/tidb/br/pkg/backup/return-region-on-find-target-peer")
}
Loading

0 comments on commit 7d9f45e

Please sign in to comment.