From 2efc9bad5d8a931c38d5b5cf3b0f2e4d2d36302b Mon Sep 17 00:00:00 2001 From: 3pointer Date: Fri, 8 Dec 2023 13:20:52 +0800 Subject: [PATCH 1/4] This is an automated cherry-pick of #48646 Signed-off-by: ti-chi-bot --- br/pkg/backup/client.go | 50 +++-------- br/pkg/backup/client_test.go | 28 ++++-- br/pkg/backup/push.go | 39 ++------- br/pkg/errors/errors.go | 2 + br/pkg/utils/BUILD.bazel | 5 ++ br/pkg/utils/backoff.go | 20 +++-- br/pkg/utils/backoff_test.go | 26 +++++- br/pkg/utils/permission.go | 6 +- br/pkg/utils/retry.go | 160 +++++++++++++++++++++++++++++++++++ br/pkg/utils/retry_test.go | 73 ++++++++++++++++ errors.toml | 10 +++ 11 files changed, 328 insertions(+), 91 deletions(-) diff --git a/br/pkg/backup/client.go b/br/pkg/backup/client.go index b7e209d92e9fb..b3191a6c28485 100644 --- a/br/pkg/backup/client.go +++ b/br/pkg/backup/client.go @@ -1110,17 +1110,18 @@ func OnBackupResponse( backupTS uint64, lockResolver *txnlock.LockResolver, resp *backuppb.BackupResponse, + errContext *utils.ErrorContext, ) (*backuppb.BackupResponse, int, error) { log.Debug("OnBackupResponse", zap.Reflect("resp", resp)) if resp.Error == nil { return resp, 0, nil } backoffMs := 0 - switch v := resp.Error.Detail.(type) { + + err := resp.Error + switch v := err.Detail.(type) { case *backuppb.Error_KvError: if lockErr := v.KvError.Locked; lockErr != nil { - // Try to resolve lock. - log.Warn("backup occur kv error", zap.Reflect("error", v)) msBeforeExpired, err1 := lockResolver.ResolveLocks( bo, backupTS, []*txnlock.Lock{txnlock.NewLock(lockErr)}) if err1 != nil { @@ -1131,44 +1132,16 @@ func OnBackupResponse( } return nil, backoffMs, nil } - // Backup should not meet error other than KeyLocked. - log.Error("unexpect kv error", zap.Reflect("KvError", v.KvError)) - return nil, backoffMs, errors.Annotatef(berrors.ErrKVUnknown, "storeID: %d OnBackupResponse error %v", storeID, v) - - case *backuppb.Error_RegionError: - regionErr := v.RegionError - // Ignore following errors. - if !(regionErr.EpochNotMatch != nil || - regionErr.NotLeader != nil || - regionErr.RegionNotFound != nil || - regionErr.ServerIsBusy != nil || - regionErr.StaleCommand != nil || - regionErr.StoreNotMatch != nil || - regionErr.ReadIndexNotReady != nil || - regionErr.ProposalInMergingMode != nil) { - log.Error("unexpect region error", zap.Reflect("RegionError", regionErr)) - return nil, backoffMs, errors.Annotatef(berrors.ErrKVUnknown, "storeID: %d OnBackupResponse error %v", storeID, v) - } - log.Warn("backup occur region error", - zap.Reflect("RegionError", regionErr), - zap.Uint64("storeID", storeID)) - // TODO: a better backoff. - backoffMs = 1000 /* 1s */ - return nil, backoffMs, nil - case *backuppb.Error_ClusterIdError: - log.Error("backup occur cluster ID error", zap.Reflect("error", v), zap.Uint64("storeID", storeID)) - return nil, 0, errors.Annotatef(berrors.ErrKVClusterIDMismatch, "%v on storeID: %d", resp.Error, storeID) default: - // UNSAFE! TODO: use meaningful error code instead of unstructured message to find failed to write error. - if utils.MessageIsRetryableStorageError(resp.GetError().GetMsg()) { - log.Warn("backup occur storage error", zap.String("error", resp.GetError().GetMsg())) - // back off 3000ms, for S3 is 99.99% available (i.e. the max outage time would less than 52.56mins per year), - // this time would be probably enough for s3 to resume. + res := errContext.HandleError(resp.Error, storeID) + switch res.Strategy { + case utils.GiveUpStrategy: + return nil, 0, errors.Annotatef(berrors.ErrKVUnknown, "storeID: %d OnBackupResponse error %s", storeID, res.Reason) + case utils.RetryStrategy: return nil, 3000, nil } - log.Error("backup occur unknown error", zap.String("error", resp.Error.GetMsg()), zap.Uint64("storeID", storeID)) - return nil, 0, errors.Annotatef(berrors.ErrKVUnknown, "%v on storeID: %d", resp.Error, storeID) } + return nil, 3000, errors.Annotatef(berrors.ErrKVUnknown, "unreachable") } func (bc *Client) handleFineGrained( @@ -1197,12 +1170,13 @@ func (bc *Client) handleFineGrained( } hasProgress := false backoffMill := 0 + errContext := utils.NewErrorContext("handleFineGrainedBackup", 10) err = SendBackup( ctx, storeID, client, req, // Handle responses with the same backoffer. func(resp *backuppb.BackupResponse) error { response, shouldBackoff, err1 := - OnBackupResponse(storeID, bo, req.EndVersion, lockResolver, resp) + OnBackupResponse(storeID, bo, req.EndVersion, lockResolver, resp, errContext) if err1 != nil { return err1 } diff --git a/br/pkg/backup/client_test.go b/br/pkg/backup/client_test.go index 592416e8ec03c..f296111789ea9 100644 --- a/br/pkg/backup/client_test.go +++ b/br/pkg/backup/client_test.go @@ -20,12 +20,22 @@ import ( "github.com/pingcap/tidb/br/pkg/mock" "github.com/pingcap/tidb/br/pkg/pdutil" "github.com/pingcap/tidb/br/pkg/storage" +<<<<<<< HEAD "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" +======= + "github.com/pingcap/tidb/br/pkg/utils" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/tablecodec" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/codec" +>>>>>>> 6c30c6e6aa7 (br: refactor error handle mechanism to tolerant unexpect kv errors. (#48646)) "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/testutils" @@ -230,20 +240,20 @@ func TestOnBackupRegionErrorResponse(t *testing.T) { } cases := []Case{ - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{NotLeader: &errorpb.NotLeader{}}), exceptedBackoffMs: 1000, exceptedErr: false}, - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{NotLeader: &errorpb.NotLeader{}}), exceptedBackoffMs: 3000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}), exceptedBackoffMs: 3000, exceptedErr: false}, {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{KeyNotInRegion: &errorpb.KeyNotInRegion{}}), exceptedBackoffMs: 0, exceptedErr: true}, - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}), exceptedBackoffMs: 1000, exceptedErr: false}, - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}), exceptedBackoffMs: 1000, exceptedErr: false}, - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{StaleCommand: &errorpb.StaleCommand{}}), exceptedBackoffMs: 1000, exceptedErr: false}, - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{StoreNotMatch: &errorpb.StoreNotMatch{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}), exceptedBackoffMs: 3000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}), exceptedBackoffMs: 3000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{StaleCommand: &errorpb.StaleCommand{}}), exceptedBackoffMs: 3000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{StoreNotMatch: &errorpb.StoreNotMatch{}}), exceptedBackoffMs: 3000, exceptedErr: false}, {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{RaftEntryTooLarge: &errorpb.RaftEntryTooLarge{}}), exceptedBackoffMs: 0, exceptedErr: true}, - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ReadIndexNotReady: &errorpb.ReadIndexNotReady{}}), exceptedBackoffMs: 1000, exceptedErr: false}, - {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ProposalInMergingMode: &errorpb.ProposalInMergingMode{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ReadIndexNotReady: &errorpb.ReadIndexNotReady{}}), exceptedBackoffMs: 3000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ProposalInMergingMode: &errorpb.ProposalInMergingMode{}}), exceptedBackoffMs: 3000, exceptedErr: false}, } for _, cs := range cases { t.Log(cs) - _, backoffMs, err := backup.OnBackupResponse(cs.storeID, cs.bo, cs.backupTS, cs.lockResolver, cs.resp) + _, backoffMs, err := backup.OnBackupResponse(cs.storeID, cs.bo, cs.backupTS, cs.lockResolver, cs.resp, utils.NewErrorContext("test", 1)) require.Equal(t, cs.exceptedBackoffMs, backoffMs) if cs.exceptedErr { require.Error(t, err) diff --git a/br/pkg/backup/push.go b/br/pkg/backup/push.go index f41b648bba52f..615765023b83b 100644 --- a/br/pkg/backup/push.go +++ b/br/pkg/backup/push.go @@ -4,7 +4,6 @@ package backup import ( "context" - "fmt" "sync" "github.com/opentracing/opentracing-go" @@ -73,6 +72,7 @@ func (push *pushDown) pushBackup( }) wg := new(sync.WaitGroup) + errContext := utils.NewErrorContext("pushBackup", 10) for _, s := range stores { store := s storeID := s.GetId() @@ -182,35 +182,10 @@ func (push *pushDown) pushBackup( progressCallBack(RegionUnit) } else { errPb := resp.GetError() - switch v := errPb.Detail.(type) { - case *backuppb.Error_KvError: - logutil.CL(ctx).Warn("backup occur kv error", zap.Reflect("error", v)) - - case *backuppb.Error_RegionError: - logutil.CL(ctx).Warn("backup occur region error", zap.Reflect("error", v)) - - case *backuppb.Error_ClusterIdError: - logutil.CL(ctx).Error("backup occur cluster ID error", zap.Reflect("error", v)) - return errors.Annotatef(berrors.ErrKVClusterIDMismatch, "%v", errPb) - default: - if utils.MessageIsRetryableStorageError(errPb.GetMsg()) { - logutil.CL(ctx).Warn("backup occur storage error", zap.String("error", errPb.GetMsg())) - continue - } - var errMsg string - if utils.MessageIsNotFoundStorageError(errPb.GetMsg()) { - errMsg = fmt.Sprintf("File or directory not found on TiKV Node (store id: %v; Address: %s). "+ - "work around:please ensure br and tikv nodes share a same storage and the user of br and tikv has same uid.", - store.GetId(), redact.String(store.GetAddress())) - logutil.CL(ctx).Error("", zap.String("error", berrors.ErrKVStorage.Error()+": "+errMsg)) - } - if utils.MessageIsPermissionDeniedStorageError(errPb.GetMsg()) { - errMsg = fmt.Sprintf("I/O permission denied error occurs on TiKV Node(store id: %v; Address: %s). "+ - "work around:please ensure tikv has permission to read from & write to the storage.", - store.GetId(), redact.String(store.GetAddress())) - logutil.CL(ctx).Error("", zap.String("error", berrors.ErrKVStorage.Error()+": "+errMsg)) - } - + res := errContext.HandleIgnorableError(errPb, store.GetId()) + switch res.Strategy { + case utils.GiveUpStrategy: + errMsg := res.Reason if len(errMsg) <= 0 { errMsg = errPb.Msg } @@ -219,6 +194,10 @@ func (push *pushDown) pushBackup( redact.String(store.GetAddress()), errMsg, ) + default: + // other type just continue for next response + // and finally handle the range in fineGrainedBackup + continue } } case err := <-push.errCh: diff --git a/br/pkg/errors/errors.go b/br/pkg/errors/errors.go index e34805d8f77dc..370cecb785110 100644 --- a/br/pkg/errors/errors.go +++ b/br/pkg/errors/errors.go @@ -49,6 +49,8 @@ var ( ErrBackupInvalidRange = errors.Normalize("backup range invalid", errors.RFCCodeText("BR:Backup:ErrBackupInvalidRange")) ErrBackupNoLeader = errors.Normalize("backup no leader", errors.RFCCodeText("BR:Backup:ErrBackupNoLeader")) ErrBackupGCSafepointExceeded = errors.Normalize("backup GC safepoint exceeded", errors.RFCCodeText("BR:Backup:ErrBackupGCSafepointExceeded")) + ErrBackupKeyIsLocked = errors.Normalize("backup key is locked", errors.RFCCodeText("BR:Backup:ErrBackupKeyIsLocked")) + ErrBackupRegion = errors.Normalize("backup region error", errors.RFCCodeText("BR:Backup:ErrBackupRegion")) ErrRestoreModeMismatch = errors.Normalize("restore mode mismatch", errors.RFCCodeText("BR:Restore:ErrRestoreModeMismatch")) ErrRestoreRangeMismatch = errors.Normalize("restore range mismatch", errors.RFCCodeText("BR:Restore:ErrRestoreRangeMismatch")) diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel index d119c77364e1b..5428514f142de 100644 --- a/br/pkg/utils/BUILD.bazel +++ b/br/pkg/utils/BUILD.bazel @@ -90,7 +90,11 @@ go_test( ], embed = [":utils"], flaky = True, +<<<<<<< HEAD shard_count = 29, +======= + shard_count = 37, +>>>>>>> 6c30c6e6aa7 (br: refactor error handle mechanism to tolerant unexpect kv errors. (#48646)) deps = [ "//br/pkg/errors", "//br/pkg/metautil", @@ -110,6 +114,7 @@ go_test( "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", "@com_github_pingcap_kvproto//pkg/encryptionpb", + "@com_github_pingcap_kvproto//pkg/errorpb", "@com_github_pingcap_kvproto//pkg/import_sstpb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_stretchr_testify//require", diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go index 1a9f157807f3e..d0a21d92b45e8 100644 --- a/br/pkg/utils/backoff.go +++ b/br/pkg/utils/backoff.go @@ -111,28 +111,34 @@ type importerBackoffer struct { attempt int delayTime time.Duration maxDelayTime time.Duration + errContext *ErrorContext } // NewBackoffer creates a new controller regulating a truncated exponential backoff. -func NewBackoffer(attempt int, delayTime, maxDelayTime time.Duration) Backoffer { +func NewBackoffer(attempt int, delayTime, maxDelayTime time.Duration, errContext *ErrorContext) Backoffer { return &importerBackoffer{ attempt: attempt, delayTime: delayTime, maxDelayTime: maxDelayTime, + errContext: errContext, } } func NewImportSSTBackoffer() Backoffer { - return NewBackoffer(importSSTRetryTimes, importSSTWaitInterval, importSSTMaxWaitInterval) + errContext := NewErrorContext("import sst", 3) + return NewBackoffer(importSSTRetryTimes, importSSTWaitInterval, importSSTMaxWaitInterval, errContext) } func NewDownloadSSTBackoffer() Backoffer { - return NewBackoffer(downloadSSTRetryTimes, downloadSSTWaitInterval, downloadSSTMaxWaitInterval) + errContext := NewErrorContext("download sst", 3) + return NewBackoffer(downloadSSTRetryTimes, downloadSSTWaitInterval, downloadSSTMaxWaitInterval, errContext) } func (bo *importerBackoffer) NextBackoff(err error) time.Duration { log.Warn("retry to import ssts", zap.Int("attempt", bo.attempt), zap.Error(err)) - if MessageIsRetryableStorageError(err.Error()) { + // we don't care storeID here. + res := bo.errContext.HandleErrorMsg(err.Error(), 0) + if res.Strategy == RetryStrategy { bo.delayTime = 2 * bo.delayTime bo.attempt-- } else { @@ -142,7 +148,7 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration { bo.delayTime = 2 * bo.delayTime bo.attempt-- case berrors.ErrKVRangeIsEmpty, berrors.ErrKVRewriteRuleNotFound: - // Excepted error, finish the operation + // Expected error, finish the operation bo.delayTime = 0 bo.attempt = 0 default: @@ -151,10 +157,10 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration { bo.delayTime = 2 * bo.delayTime bo.attempt-- default: - // Unexcepted error + // Unexpected error bo.delayTime = 0 bo.attempt = 0 - log.Warn("unexcepted error, stop to retry", zap.Error(err)) + log.Warn("unexpected error, stop retrying", zap.Error(err)) } } } diff --git a/br/pkg/utils/backoff_test.go b/br/pkg/utils/backoff_test.go index 31778052f77e1..dc09826fd7806 100644 --- a/br/pkg/utils/backoff_test.go +++ b/br/pkg/utils/backoff_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/pingcap/errors" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/utils" "github.com/stretchr/testify/require" @@ -18,7 +19,7 @@ import ( func TestBackoffWithSuccess(t *testing.T) { var counter int - backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond) + backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond, utils.NewDefaultContext()) err := utils.WithRetry(context.Background(), func() error { defer func() { counter++ }() switch counter { @@ -35,9 +36,26 @@ func TestBackoffWithSuccess(t *testing.T) { require.NoError(t, err) } +func TestBackoffWithUnknowneErrorSuccess(t *testing.T) { + var counter int + backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond, utils.NewDefaultContext()) + err := utils.WithRetry(context.Background(), func() error { + defer func() { counter++ }() + switch counter { + case 0: + return errors.New("unknown error: not in the allow list") + case 1: + return berrors.ErrKVEpochNotMatch + } + return nil + }, backoffer) + require.Equal(t, 3, counter) + require.NoError(t, err) +} + func TestBackoffWithFatalError(t *testing.T) { var counter int - backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond) + backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond, utils.NewDefaultContext()) gRPCError := status.Error(codes.Unavailable, "transport is closing") err := utils.WithRetry(context.Background(), func() error { defer func() { counter++ }() @@ -65,7 +83,7 @@ func TestBackoffWithFatalError(t *testing.T) { func TestBackoffWithFatalRawGRPCError(t *testing.T) { var counter int canceledError := status.Error(codes.Canceled, "context canceled") - backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond) + backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond, utils.NewDefaultContext()) err := utils.WithRetry(context.Background(), func() error { defer func() { counter++ }() return canceledError // nolint:wrapcheck @@ -76,7 +94,7 @@ func TestBackoffWithFatalRawGRPCError(t *testing.T) { func TestBackoffWithRetryableError(t *testing.T) { var counter int - backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond) + backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond, utils.NewDefaultContext()) err := utils.WithRetry(context.Background(), func() error { defer func() { counter++ }() return berrors.ErrKVEpochNotMatch diff --git a/br/pkg/utils/permission.go b/br/pkg/utils/permission.go index e18c28dbbbe1c..3c0795db11c47 100644 --- a/br/pkg/utils/permission.go +++ b/br/pkg/utils/permission.go @@ -7,14 +7,14 @@ var ( permissionDeniedMsg = "permissiondenied" ) -// MessageIsNotFoundStorageError checks whether the message returning from TiKV is "NotFound" storage I/O error -func MessageIsNotFoundStorageError(msg string) bool { +// messageIsNotFoundStorageError checks whether the message returning from TiKV is "NotFound" storage I/O error +func messageIsNotFoundStorageError(msg string) bool { msgLower := strings.ToLower(msg) return strings.Contains(msgLower, "io") && strings.Contains(msgLower, ioNotFoundMsg) } // MessageIsPermissionDeniedStorageError checks whether the message returning from TiKV is "PermissionDenied" storage I/O error -func MessageIsPermissionDeniedStorageError(msg string) bool { +func messageIsPermissionDeniedStorageError(msg string) bool { msgLower := strings.ToLower(msg) return strings.Contains(msgLower, permissionDeniedMsg) } diff --git a/br/pkg/utils/retry.go b/br/pkg/utils/retry.go index 20482d7c423a2..0baf07a6cf555 100644 --- a/br/pkg/utils/retry.go +++ b/br/pkg/utils/retry.go @@ -4,16 +4,25 @@ package utils import ( "context" + "fmt" "strings" "sync" "time" "github.com/cznic/mathutil" "github.com/pingcap/errors" +<<<<<<< HEAD tmysql "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/parser/terror" +======= + backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/log" + tmysql "github.com/pingcap/tidb/pkg/errno" + "github.com/pingcap/tidb/pkg/parser/terror" +>>>>>>> 6c30c6e6aa7 (br: refactor error handle mechanism to tolerant unexpect kv errors. (#48646)) "github.com/tikv/client-go/v2/tikv" "go.uber.org/multierr" + "go.uber.org/zap" ) var retryableServerError = []string{ @@ -33,6 +42,157 @@ var retryableServerError = []string{ "end of file before message length reached", } +type ErrorResult struct { + Strategy ErrorStrategy + Reason string +} + +type ErrorStrategy int + +const ( + // This type can be retry but consume the backoffer attempts. + RetryStrategy ErrorStrategy = iota + // This type means unrecoverable error and the whole progress should exits + // for example: + // 1. permission not valid. + // 2. data has not found. + // 3. retry too many times + GiveUpStrategy + // This type represents Unknown error + UnknownStrategy +) + +type ErrorContext struct { + mu sync.Mutex + // encounter times for one context on a store + // we may use this value to determine the retry policy + encounterTimes map[uint64]int + // unknown error retry limitation. + // encouter many times error makes Retry to GiveUp. + encounterTimesLimitation int + // whether in backup or restore + scenario string +} + +func NewErrorContext(scenario string, limitation int) *ErrorContext { + return &ErrorContext{ + scenario: scenario, + encounterTimes: make(map[uint64]int), + encounterTimesLimitation: limitation, + } +} + +func NewDefaultContext() *ErrorContext { + return &ErrorContext{ + scenario: "default", + encounterTimes: make(map[uint64]int), + encounterTimesLimitation: 1, + } +} + +func (ec *ErrorContext) HandleError(err *backuppb.Error, uuid uint64) ErrorResult { + if err == nil { + return ErrorResult{RetryStrategy, "unreachable retry"} + } + res := ec.handleErrorPb(err, uuid) + // try the best effort to save progress from error here + if res.Strategy == UnknownStrategy && len(err.Msg) != 0 { + return ec.HandleErrorMsg(err.Msg, uuid) + } + return res +} + +func (ec *ErrorContext) HandleIgnorableError(err *backuppb.Error, uuid uint64) ErrorResult { + if err == nil { + return ErrorResult{RetryStrategy, "unreachable retry"} + } + res := ec.handleIgnorableErrorPb(err, uuid) + // try the best effort to save progress from error here + if res.Strategy == UnknownStrategy && len(err.Msg) != 0 { + return ec.HandleErrorMsg(err.Msg, uuid) + } + return res +} + +func (ec *ErrorContext) HandleErrorMsg(msg string, uuid uint64) ErrorResult { + // UNSAFE! TODO: use meaningful error code instead of unstructured message to find failed to write error. + logger := log.L().With(zap.String("scenario", ec.scenario)) + if messageIsNotFoundStorageError(msg) { + reason := fmt.Sprintf("File or directory not found on TiKV Node (store id: %v). "+ + "work around:please ensure br and tikv nodes share a same storage and the user of br and tikv has same uid.", + uuid) + return ErrorResult{GiveUpStrategy, reason} + } + if messageIsPermissionDeniedStorageError(msg) { + reason := fmt.Sprintf("I/O permission denied error occurs on TiKV Node(store id: %v). "+ + "work around:please ensure tikv has permission to read from & write to the storage.", + uuid) + return ErrorResult{GiveUpStrategy, reason} + } + msgLower := strings.ToLower(msg) + if strings.Contains(msgLower, "context canceled") { + return ErrorResult{GiveUpStrategy, "context canceled, give up"} + } + + if MessageIsRetryableStorageError(msg) { + logger.Warn("occur storage error", zap.String("error", msg)) + return ErrorResult{RetryStrategy, "retrable error"} + } + // retry enough on same store + ec.mu.Lock() + defer ec.mu.Unlock() + ec.encounterTimes[uuid]++ + if ec.encounterTimes[uuid] <= ec.encounterTimesLimitation { + return ErrorResult{RetryStrategy, "unknown error, retry it for few times"} + } + return ErrorResult{GiveUpStrategy, "unknown error and retry too many times, give up"} +} + +func (ec *ErrorContext) handleIgnorableErrorPb(e *backuppb.Error, uuid uint64) ErrorResult { + switch e.Detail.(type) { + case *backuppb.Error_KvError: + return ErrorResult{RetryStrategy, "retry outside because the error can be ignored"} + case *backuppb.Error_RegionError: + return ErrorResult{RetryStrategy, "retry outside because the error can be ignored"} + case *backuppb.Error_ClusterIdError: + return ErrorResult{GiveUpStrategy, "cluster ID mismatch"} + } + return ErrorResult{UnknownStrategy, "unreachable code"} +} + +func (ec *ErrorContext) handleErrorPb(e *backuppb.Error, uuid uint64) ErrorResult { + logger := log.L().With(zap.String("scenario", ec.scenario)) + switch v := e.Detail.(type) { + case *backuppb.Error_KvError: + // should not meet error other than KeyLocked. + return ErrorResult{GiveUpStrategy, "unknown kv error"} + + case *backuppb.Error_RegionError: + regionErr := v.RegionError + // Ignore following errors. + if !(regionErr.EpochNotMatch != nil || + regionErr.NotLeader != nil || + regionErr.RegionNotFound != nil || + regionErr.ServerIsBusy != nil || + regionErr.StaleCommand != nil || + regionErr.StoreNotMatch != nil || + regionErr.ReadIndexNotReady != nil || + regionErr.ProposalInMergingMode != nil) { + logger.Error("unexpect region error", zap.Reflect("RegionError", regionErr)) + return ErrorResult{GiveUpStrategy, "unknown kv error"} + } + logger.Warn("occur region error", + zap.Reflect("RegionError", regionErr), + zap.Uint64("uuid", uuid)) + return ErrorResult{RetryStrategy, "retrable error"} + + case *backuppb.Error_ClusterIdError: + logger.Error("occur cluster ID error", zap.Reflect("error", v), zap.Uint64("uuid", uuid)) + return ErrorResult{GiveUpStrategy, "cluster ID mismatch"} + } + return ErrorResult{UnknownStrategy, "unreachable code"} +} + // RetryableFunc presents a retryable operation. type RetryableFunc func() error diff --git a/br/pkg/utils/retry_test.go b/br/pkg/utils/retry_test.go index eeef8c61c0480..eeaef51ab8dfd 100644 --- a/br/pkg/utils/retry_test.go +++ b/br/pkg/utils/retry_test.go @@ -9,6 +9,8 @@ import ( "time" "github.com/pingcap/errors" + backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/tidb/br/pkg/utils" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" @@ -47,3 +49,74 @@ func TestRetryAdapter(t *testing.T) { req.Greater(time.Since(begin), 200*time.Millisecond) } + +func TestHandleError(t *testing.T) { + ec := utils.NewErrorContext("test", 3) + // Test case 1: Error is nil + result := ec.HandleError(nil, 123) + require.Equal(t, utils.ErrorResult{utils.RetryStrategy, "unreachable retry"}, result) + + // Test case 2: Error is KvError and can be ignored + kvError := &backuppb.Error_KvError{} + result = ec.HandleIgnorableError(&backuppb.Error{Detail: kvError}, 123) + require.Equal(t, utils.ErrorResult{utils.RetryStrategy, "retry outside because the error can be ignored"}, result) + + // Test case 3: Error is KvError and cannot be ignored + result = ec.HandleError(&backuppb.Error{Detail: kvError}, 123) + require.Equal(t, utils.ErrorResult{utils.GiveUpStrategy, "unknown kv error"}, result) + + // Test case 4: Error is RegionError and can be ignored + regionError := &backuppb.Error_RegionError{ + RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{RegionId: 1}}} + result = ec.HandleIgnorableError(&backuppb.Error{Detail: regionError}, 123) + require.Equal(t, utils.ErrorResult{utils.RetryStrategy, "retry outside because the error can be ignored"}, result) + + // Test case 5: Error is RegionError and cannot be ignored + regionError = &backuppb.Error_RegionError{ + RegionError: &errorpb.Error{DiskFull: &errorpb.DiskFull{}}} + result = ec.HandleError(&backuppb.Error{Detail: regionError}, 123) + require.Equal(t, utils.ErrorResult{utils.GiveUpStrategy, "unknown kv error"}, result) + + // Test case 6: Error is ClusterIdError + clusterIdError := &backuppb.Error_ClusterIdError{} + result = ec.HandleError(&backuppb.Error{Detail: clusterIdError}, 123) + require.Equal(t, utils.ErrorResult{utils.GiveUpStrategy, "cluster ID mismatch"}, result) +} + +func TestHandleErrorMsg(t *testing.T) { + ec := utils.NewErrorContext("test", 3) + + // Test messageIsNotFoundStorageError + msg := "IO: files Notfound error" + uuid := uint64(456) + expectedReason := "File or directory not found on TiKV Node (store id: 456). work around:please ensure br and tikv nodes share a same storage and the user of br and tikv has same uid." + expectedResult := utils.ErrorResult{utils.GiveUpStrategy, expectedReason} + actualResult := ec.HandleErrorMsg(msg, uuid) + require.Equal(t, expectedResult, actualResult) + + // Test messageIsPermissionDeniedStorageError + msg = "I/O permissiondenied error occurs on TiKV Node(store id: 456)." + expectedReason = "I/O permission denied error occurs on TiKV Node(store id: 456). work around:please ensure tikv has permission to read from & write to the storage." + expectedResult = utils.ErrorResult{utils.GiveUpStrategy, expectedReason} + actualResult = ec.HandleErrorMsg(msg, uuid) + require.Equal(t, expectedResult, actualResult) + + // Test MessageIsRetryableStorageError + msg = "server closed" + expectedResult = utils.ErrorResult{utils.RetryStrategy, "retrable error"} + actualResult = ec.HandleErrorMsg(msg, uuid) + require.Equal(t, expectedResult, actualResult) + + // Test unknown error + msg = "unknown error" + expectedResult = utils.ErrorResult{utils.RetryStrategy, "unknown error, retry it for few times"} + actualResult = ec.HandleErrorMsg(msg, uuid) + require.Equal(t, expectedResult, actualResult) + + // Test retry too many times + _ = ec.HandleErrorMsg(msg, uuid) + _ = ec.HandleErrorMsg(msg, uuid) + expectedResult = utils.ErrorResult{utils.GiveUpStrategy, "unknown error and retry too many times, give up"} + actualResult = ec.HandleErrorMsg(msg, uuid) + require.Equal(t, expectedResult, actualResult) +} diff --git a/errors.toml b/errors.toml index 235a8a9375977..cf529842919f0 100644 --- a/errors.toml +++ b/errors.toml @@ -16,11 +16,21 @@ error = ''' backup range invalid ''' +["BR:Backup:ErrBackupKeyIsLocked"] +error = ''' +backup key is locked +''' + ["BR:Backup:ErrBackupNoLeader"] error = ''' backup no leader ''' +["BR:Backup:ErrBackupRegion"] +error = ''' +backup region error +''' + ["BR:Common:ErrEnvNotSpecified"] error = ''' environment variable not found From 74d445c83c3dc26ab8f97dbb58444c139bab3463 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Mon, 29 Jan 2024 11:56:25 +0800 Subject: [PATCH 2/4] resolve_conflict --- br/cmd/br/BUILD.bazel | 1 - br/pkg/aws/BUILD.bazel | 1 - br/pkg/backup/BUILD.bazel | 2 -- br/pkg/backup/client_test.go | 11 +---------- br/pkg/checkpoint/BUILD.bazel | 1 - br/pkg/conn/BUILD.bazel | 2 -- br/pkg/glue/BUILD.bazel | 1 - br/pkg/gluetikv/BUILD.bazel | 1 - br/pkg/lightning/BUILD.bazel | 1 - br/pkg/lightning/backend/kv/BUILD.bazel | 1 - br/pkg/lightning/backend/local/BUILD.bazel | 2 -- br/pkg/lightning/backend/tidb/BUILD.bazel | 1 - br/pkg/lightning/common/BUILD.bazel | 1 - br/pkg/lightning/errormanager/BUILD.bazel | 2 -- br/pkg/lightning/restore/BUILD.bazel | 1 - br/pkg/restore/BUILD.bazel | 2 -- br/pkg/restore/split/BUILD.bazel | 2 -- br/pkg/restore/tiflashrec/BUILD.bazel | 1 - br/pkg/stream/BUILD.bazel | 1 - br/pkg/streamhelper/BUILD.bazel | 2 -- br/pkg/streamhelper/spans/BUILD.bazel | 1 - br/pkg/task/BUILD.bazel | 2 -- br/pkg/task/operator/BUILD.bazel | 1 - br/pkg/utils/iter/BUILD.bazel | 1 - br/pkg/utils/retry.go | 9 ++------- br/pkg/version/BUILD.bazel | 1 - ddl/BUILD.bazel | 1 - dumpling/export/BUILD.bazel | 1 - telemetry/BUILD.bazel | 1 - 29 files changed, 3 insertions(+), 52 deletions(-) diff --git a/br/cmd/br/BUILD.bazel b/br/cmd/br/BUILD.bazel index e2921d867640b..13bd7e0e9d4b0 100644 --- a/br/cmd/br/BUILD.bazel +++ b/br/cmd/br/BUILD.bazel @@ -29,7 +29,6 @@ go_library( "//br/pkg/task", "//br/pkg/task/operator", "//br/pkg/trace", - "//br/pkg/utils", "//br/pkg/version/build", "//config", "//parser/model", diff --git a/br/pkg/aws/BUILD.bazel b/br/pkg/aws/BUILD.bazel index d250a7e757de6..1cbb17ca8f7f1 100644 --- a/br/pkg/aws/BUILD.bazel +++ b/br/pkg/aws/BUILD.bazel @@ -8,7 +8,6 @@ go_library( deps = [ "//br/pkg/config", "//br/pkg/glue", - "//br/pkg/utils", "@com_github_aws_aws_sdk_go//aws", "@com_github_aws_aws_sdk_go//aws/awserr", "@com_github_aws_aws_sdk_go//aws/session", diff --git a/br/pkg/backup/BUILD.bazel b/br/pkg/backup/BUILD.bazel index a80396f4fddd2..a98cc0321feff 100644 --- a/br/pkg/backup/BUILD.bazel +++ b/br/pkg/backup/BUILD.bazel @@ -24,7 +24,6 @@ go_library( "//br/pkg/rtree", "//br/pkg/storage", "//br/pkg/summary", - "//br/pkg/utils", "//br/pkg/version", "//ddl", "//distsql", @@ -75,7 +74,6 @@ go_test( "//br/pkg/mock", "//br/pkg/pdutil", "//br/pkg/storage", - "//br/pkg/utils", "//kv", "//parser/model", "//sessionctx/variable", diff --git a/br/pkg/backup/client_test.go b/br/pkg/backup/client_test.go index f296111789ea9..a3a442a52afcc 100644 --- a/br/pkg/backup/client_test.go +++ b/br/pkg/backup/client_test.go @@ -20,22 +20,13 @@ import ( "github.com/pingcap/tidb/br/pkg/mock" "github.com/pingcap/tidb/br/pkg/pdutil" "github.com/pingcap/tidb/br/pkg/storage" -<<<<<<< HEAD + "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" -======= - "github.com/pingcap/tidb/br/pkg/utils" - "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/parser/model" - "github.com/pingcap/tidb/pkg/tablecodec" - "github.com/pingcap/tidb/pkg/testkit" - "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/codec" ->>>>>>> 6c30c6e6aa7 (br: refactor error handle mechanism to tolerant unexpect kv errors. (#48646)) "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/testutils" diff --git a/br/pkg/checkpoint/BUILD.bazel b/br/pkg/checkpoint/BUILD.bazel index 20e39dc39025b..050f63f9650aa 100644 --- a/br/pkg/checkpoint/BUILD.bazel +++ b/br/pkg/checkpoint/BUILD.bazel @@ -11,7 +11,6 @@ go_library( "//br/pkg/rtree", "//br/pkg/storage", "//br/pkg/summary", - "//br/pkg/utils", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", "@com_github_pingcap_log//:log", diff --git a/br/pkg/conn/BUILD.bazel b/br/pkg/conn/BUILD.bazel index fc88f174394f3..b05e58a2d879e 100644 --- a/br/pkg/conn/BUILD.bazel +++ b/br/pkg/conn/BUILD.bazel @@ -11,7 +11,6 @@ go_library( "//br/pkg/glue", "//br/pkg/logutil", "//br/pkg/pdutil", - "//br/pkg/utils", "//br/pkg/version", "//domain", "//kv", @@ -47,7 +46,6 @@ go_test( deps = [ "//br/pkg/conn/util", "//br/pkg/pdutil", - "//br/pkg/utils", "//testkit/testsetup", "@com_github_docker_go_units//:go-units", "@com_github_pingcap_errors//:errors", diff --git a/br/pkg/glue/BUILD.bazel b/br/pkg/glue/BUILD.bazel index dc993f47c31d8..f20f1c0bfc165 100644 --- a/br/pkg/glue/BUILD.bazel +++ b/br/pkg/glue/BUILD.bazel @@ -10,7 +10,6 @@ go_library( importpath = "github.com/pingcap/tidb/br/pkg/glue", visibility = ["//visibility:public"], deps = [ - "//br/pkg/utils", "//ddl", "//domain", "//kv", diff --git a/br/pkg/gluetikv/BUILD.bazel b/br/pkg/gluetikv/BUILD.bazel index 2c82b6d85b40f..b762691a3a129 100644 --- a/br/pkg/gluetikv/BUILD.bazel +++ b/br/pkg/gluetikv/BUILD.bazel @@ -8,7 +8,6 @@ go_library( deps = [ "//br/pkg/glue", "//br/pkg/summary", - "//br/pkg/utils", "//br/pkg/version/build", "//config", "//domain", diff --git a/br/pkg/lightning/BUILD.bazel b/br/pkg/lightning/BUILD.bazel index 95aca448f786a..7b201540598b1 100644 --- a/br/pkg/lightning/BUILD.bazel +++ b/br/pkg/lightning/BUILD.bazel @@ -24,7 +24,6 @@ go_library( "//br/pkg/lightning/web", "//br/pkg/redact", "//br/pkg/storage", - "//br/pkg/utils", "//br/pkg/version/build", "//expression", "//planner/core", diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel index 1696195d4abe4..ac216cd0f47c5 100644 --- a/br/pkg/lightning/backend/kv/BUILD.bazel +++ b/br/pkg/lightning/backend/kv/BUILD.bazel @@ -19,7 +19,6 @@ go_library( "//br/pkg/lightning/verification", "//br/pkg/logutil", "//br/pkg/redact", - "//br/pkg/utils", "//expression", "//kv", "//meta/autoid", diff --git a/br/pkg/lightning/backend/local/BUILD.bazel b/br/pkg/lightning/backend/local/BUILD.bazel index 9524ab5febc2b..9ca04cee8906d 100644 --- a/br/pkg/lightning/backend/local/BUILD.bazel +++ b/br/pkg/lightning/backend/local/BUILD.bazel @@ -33,7 +33,6 @@ go_library( "//br/pkg/membuf", "//br/pkg/pdutil", "//br/pkg/restore/split", - "//br/pkg/utils", "//br/pkg/version", "//distsql", "//infoschema", @@ -108,7 +107,6 @@ go_test( "//br/pkg/mock", "//br/pkg/pdutil", "//br/pkg/restore/split", - "//br/pkg/utils", "//ddl", "//kv", "//parser", diff --git a/br/pkg/lightning/backend/tidb/BUILD.bazel b/br/pkg/lightning/backend/tidb/BUILD.bazel index 9dd8a9a876c14..5e6f3c1a546d2 100644 --- a/br/pkg/lightning/backend/tidb/BUILD.bazel +++ b/br/pkg/lightning/backend/tidb/BUILD.bazel @@ -14,7 +14,6 @@ go_library( "//br/pkg/lightning/log", "//br/pkg/lightning/verification", "//br/pkg/redact", - "//br/pkg/utils", "//br/pkg/version", "//parser/model", "//parser/mysql", diff --git a/br/pkg/lightning/common/BUILD.bazel b/br/pkg/lightning/common/BUILD.bazel index 964e01f31d4fa..9b19f08cebd06 100644 --- a/br/pkg/lightning/common/BUILD.bazel +++ b/br/pkg/lightning/common/BUILD.bazel @@ -20,7 +20,6 @@ go_library( "//br/pkg/errors", "//br/pkg/httputil", "//br/pkg/lightning/log", - "//br/pkg/utils", "//errno", "//parser/model", "//store/driver/error", diff --git a/br/pkg/lightning/errormanager/BUILD.bazel b/br/pkg/lightning/errormanager/BUILD.bazel index c557eca4c7ce4..855c13d6eed41 100644 --- a/br/pkg/lightning/errormanager/BUILD.bazel +++ b/br/pkg/lightning/errormanager/BUILD.bazel @@ -10,7 +10,6 @@ go_library( "//br/pkg/lightning/config", "//br/pkg/lightning/log", "//br/pkg/redact", - "//br/pkg/utils", "@com_github_jedib0t_go_pretty_v6//table", "@com_github_jedib0t_go_pretty_v6//text", "@com_github_pingcap_errors//:errors", @@ -29,7 +28,6 @@ go_test( deps = [ "//br/pkg/lightning/config", "//br/pkg/lightning/log", - "//br/pkg/utils", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_stretchr_testify//require", "@org_uber_go_atomic//:atomic", diff --git a/br/pkg/lightning/restore/BUILD.bazel b/br/pkg/lightning/restore/BUILD.bazel index 3c2be48b86e19..c37d3a77b20c6 100644 --- a/br/pkg/lightning/restore/BUILD.bazel +++ b/br/pkg/lightning/restore/BUILD.bazel @@ -40,7 +40,6 @@ go_library( "//br/pkg/redact", "//br/pkg/storage", "//br/pkg/streamhelper", - "//br/pkg/utils", "//br/pkg/version", "//br/pkg/version/build", "//config", diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel index 85f0836e28d9f..8da6cc8bf3f3f 100644 --- a/br/pkg/restore/BUILD.bazel +++ b/br/pkg/restore/BUILD.bazel @@ -42,7 +42,6 @@ go_library( "//br/pkg/storage", "//br/pkg/stream", "//br/pkg/summary", - "//br/pkg/utils", "//br/pkg/utils/iter", "//br/pkg/utils/storewatch", "//br/pkg/version", @@ -141,7 +140,6 @@ go_test( "//br/pkg/rtree", "//br/pkg/storage", "//br/pkg/stream", - "//br/pkg/utils", "//br/pkg/utils/iter", "//infoschema", "//kv", diff --git a/br/pkg/restore/split/BUILD.bazel b/br/pkg/restore/split/BUILD.bazel index 920f8ec302267..1ae186e9f22cc 100644 --- a/br/pkg/restore/split/BUILD.bazel +++ b/br/pkg/restore/split/BUILD.bazel @@ -15,7 +15,6 @@ go_library( "//br/pkg/httputil", "//br/pkg/logutil", "//br/pkg/redact", - "//br/pkg/utils", "//store/pdtypes", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", @@ -43,7 +42,6 @@ go_test( deps = [ ":split", "//br/pkg/errors", - "//br/pkg/utils", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", ], diff --git a/br/pkg/restore/tiflashrec/BUILD.bazel b/br/pkg/restore/tiflashrec/BUILD.bazel index fbe6ba415cb8c..16ffa1b2564ed 100644 --- a/br/pkg/restore/tiflashrec/BUILD.bazel +++ b/br/pkg/restore/tiflashrec/BUILD.bazel @@ -7,7 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//br/pkg/logutil", - "//br/pkg/utils", "//infoschema", "//parser/ast", "//parser/format", diff --git a/br/pkg/stream/BUILD.bazel b/br/pkg/stream/BUILD.bazel index f75f9f37d81ea..cb3d585dc8242 100644 --- a/br/pkg/stream/BUILD.bazel +++ b/br/pkg/stream/BUILD.bazel @@ -19,7 +19,6 @@ go_library( "//br/pkg/logutil", "//br/pkg/storage", "//br/pkg/streamhelper", - "//br/pkg/utils", "//kv", "//meta", "//parser/model", diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel index 19926e96aa0df..027c2d0fec547 100644 --- a/br/pkg/streamhelper/BUILD.bazel +++ b/br/pkg/streamhelper/BUILD.bazel @@ -22,7 +22,6 @@ go_library( "//br/pkg/redact", "//br/pkg/streamhelper/config", "//br/pkg/streamhelper/spans", - "//br/pkg/utils", "//config", "//kv", "//metrics", @@ -77,7 +76,6 @@ go_test( "//br/pkg/storage", "//br/pkg/streamhelper/config", "//br/pkg/streamhelper/spans", - "//br/pkg/utils", "//kv", "//tablecodec", "//util/codec", diff --git a/br/pkg/streamhelper/spans/BUILD.bazel b/br/pkg/streamhelper/spans/BUILD.bazel index 899f6f6ade6b1..a8de313029f43 100644 --- a/br/pkg/streamhelper/spans/BUILD.bazel +++ b/br/pkg/streamhelper/spans/BUILD.bazel @@ -11,7 +11,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//br/pkg/logutil", - "//br/pkg/utils", "//kv", "@com_github_google_btree//:btree", ], diff --git a/br/pkg/task/BUILD.bazel b/br/pkg/task/BUILD.bazel index 219ea0d304aa8..7acdbdda9501c 100644 --- a/br/pkg/task/BUILD.bazel +++ b/br/pkg/task/BUILD.bazel @@ -38,7 +38,6 @@ go_library( "//br/pkg/streamhelper/config", "//br/pkg/streamhelper/daemon", "//br/pkg/summary", - "//br/pkg/utils", "//br/pkg/version", "//config", "//ddl", @@ -101,7 +100,6 @@ go_test( "//br/pkg/restore", "//br/pkg/storage", "//br/pkg/stream", - "//br/pkg/utils", "//config", "//parser/model", "//statistics/handle", diff --git a/br/pkg/task/operator/BUILD.bazel b/br/pkg/task/operator/BUILD.bazel index 5ce85cbd1313f..b69fb7d48211d 100644 --- a/br/pkg/task/operator/BUILD.bazel +++ b/br/pkg/task/operator/BUILD.bazel @@ -12,7 +12,6 @@ go_library( "//br/pkg/logutil", "//br/pkg/pdutil", "//br/pkg/task", - "//br/pkg/utils", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_log//:log", "@com_github_spf13_pflag//:pflag", diff --git a/br/pkg/utils/iter/BUILD.bazel b/br/pkg/utils/iter/BUILD.bazel index 0e4c55ed67d56..4c39e57a0257b 100644 --- a/br/pkg/utils/iter/BUILD.bazel +++ b/br/pkg/utils/iter/BUILD.bazel @@ -12,7 +12,6 @@ go_library( importpath = "github.com/pingcap/tidb/br/pkg/utils/iter", visibility = ["//visibility:public"], deps = [ - "//br/pkg/utils", "@org_golang_x_exp//constraints", "@org_golang_x_sync//errgroup", ], diff --git a/br/pkg/utils/retry.go b/br/pkg/utils/retry.go index 0baf07a6cf555..3a3e1879276a2 100644 --- a/br/pkg/utils/retry.go +++ b/br/pkg/utils/retry.go @@ -11,15 +11,10 @@ import ( "github.com/cznic/mathutil" "github.com/pingcap/errors" -<<<<<<< HEAD - tmysql "github.com/pingcap/tidb/errno" - "github.com/pingcap/tidb/parser/terror" -======= backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/log" - tmysql "github.com/pingcap/tidb/pkg/errno" - "github.com/pingcap/tidb/pkg/parser/terror" ->>>>>>> 6c30c6e6aa7 (br: refactor error handle mechanism to tolerant unexpect kv errors. (#48646)) + tmysql "github.com/pingcap/tidb/errno" + "github.com/pingcap/tidb/parser/terror" "github.com/tikv/client-go/v2/tikv" "go.uber.org/multierr" "go.uber.org/zap" diff --git a/br/pkg/version/BUILD.bazel b/br/pkg/version/BUILD.bazel index f969242b7aeaa..5d759bc6a404b 100644 --- a/br/pkg/version/BUILD.bazel +++ b/br/pkg/version/BUILD.bazel @@ -8,7 +8,6 @@ go_library( deps = [ "//br/pkg/errors", "//br/pkg/logutil", - "//br/pkg/utils", "//br/pkg/version/build", "//parser/model", "//sessionctx/variable", diff --git a/ddl/BUILD.bazel b/ddl/BUILD.bazel index db38aca5ec021..7965daa8e3b0c 100644 --- a/ddl/BUILD.bazel +++ b/ddl/BUILD.bazel @@ -53,7 +53,6 @@ go_library( ], deps = [ "//br/pkg/lightning/common", - "//br/pkg/utils", "//config", "//ddl/ingest", "//ddl/label", diff --git a/dumpling/export/BUILD.bazel b/dumpling/export/BUILD.bazel index fc3006cb855c0..274fb6315289b 100644 --- a/dumpling/export/BUILD.bazel +++ b/dumpling/export/BUILD.bazel @@ -28,7 +28,6 @@ go_library( deps = [ "//br/pkg/storage", "//br/pkg/summary", - "//br/pkg/utils", "//br/pkg/version", "//config", "//dumpling/cli", diff --git a/telemetry/BUILD.bazel b/telemetry/BUILD.bazel index 1f032aa3f237a..3d0a0a8e7004e 100644 --- a/telemetry/BUILD.bazel +++ b/telemetry/BUILD.bazel @@ -18,7 +18,6 @@ go_library( importpath = "github.com/pingcap/tidb/telemetry", visibility = ["//visibility:public"], deps = [ - "//br/pkg/utils", "//config", "//domain/infosync", "//infoschema", From 95fa893c3462f10828ce78d332c51b3160b96d45 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Mon, 29 Jan 2024 14:12:32 +0800 Subject: [PATCH 3/4] resolve conflicdt --- br/cmd/br/BUILD.bazel | 1 + br/pkg/aws/BUILD.bazel | 1 + br/pkg/backup/BUILD.bazel | 2 ++ br/pkg/backup/prepare_snap/BUILD.bazel | 2 ++ br/pkg/checkpoint/BUILD.bazel | 1 + br/pkg/conn/BUILD.bazel | 2 ++ br/pkg/glue/BUILD.bazel | 1 + br/pkg/gluetikv/BUILD.bazel | 1 + br/pkg/lightning/BUILD.bazel | 1 + br/pkg/lightning/backend/kv/BUILD.bazel | 1 + br/pkg/lightning/backend/local/BUILD.bazel | 2 ++ br/pkg/lightning/backend/tidb/BUILD.bazel | 1 + br/pkg/lightning/common/BUILD.bazel | 1 + br/pkg/lightning/errormanager/BUILD.bazel | 2 ++ br/pkg/lightning/restore/BUILD.bazel | 1 + br/pkg/restore/BUILD.bazel | 2 ++ br/pkg/restore/split/BUILD.bazel | 2 ++ br/pkg/restore/tiflashrec/BUILD.bazel | 1 + br/pkg/stream/BUILD.bazel | 1 + br/pkg/streamhelper/BUILD.bazel | 2 ++ br/pkg/streamhelper/spans/BUILD.bazel | 1 + br/pkg/task/BUILD.bazel | 2 ++ br/pkg/task/operator/BUILD.bazel | 1 + br/pkg/utils/BUILD.bazel | 13 ------------- br/pkg/utils/iter/BUILD.bazel | 1 + br/pkg/utils/retry.go | 1 + br/pkg/version/BUILD.bazel | 1 + ddl/BUILD.bazel | 1 + dumpling/export/BUILD.bazel | 1 + telemetry/BUILD.bazel | 1 + 30 files changed, 38 insertions(+), 13 deletions(-) diff --git a/br/cmd/br/BUILD.bazel b/br/cmd/br/BUILD.bazel index 13bd7e0e9d4b0..e2921d867640b 100644 --- a/br/cmd/br/BUILD.bazel +++ b/br/cmd/br/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "//br/pkg/task", "//br/pkg/task/operator", "//br/pkg/trace", + "//br/pkg/utils", "//br/pkg/version/build", "//config", "//parser/model", diff --git a/br/pkg/aws/BUILD.bazel b/br/pkg/aws/BUILD.bazel index 1cbb17ca8f7f1..d250a7e757de6 100644 --- a/br/pkg/aws/BUILD.bazel +++ b/br/pkg/aws/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//br/pkg/config", "//br/pkg/glue", + "//br/pkg/utils", "@com_github_aws_aws_sdk_go//aws", "@com_github_aws_aws_sdk_go//aws/awserr", "@com_github_aws_aws_sdk_go//aws/session", diff --git a/br/pkg/backup/BUILD.bazel b/br/pkg/backup/BUILD.bazel index a98cc0321feff..a80396f4fddd2 100644 --- a/br/pkg/backup/BUILD.bazel +++ b/br/pkg/backup/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "//br/pkg/rtree", "//br/pkg/storage", "//br/pkg/summary", + "//br/pkg/utils", "//br/pkg/version", "//ddl", "//distsql", @@ -74,6 +75,7 @@ go_test( "//br/pkg/mock", "//br/pkg/pdutil", "//br/pkg/storage", + "//br/pkg/utils", "//kv", "//parser/model", "//sessionctx/variable", diff --git a/br/pkg/backup/prepare_snap/BUILD.bazel b/br/pkg/backup/prepare_snap/BUILD.bazel index 705ca95df5e56..43ef1e611bc3a 100644 --- a/br/pkg/backup/prepare_snap/BUILD.bazel +++ b/br/pkg/backup/prepare_snap/BUILD.bazel @@ -12,6 +12,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//br/pkg/logutil", + "//br/pkg/utils", "//util/engine", "@com_github_docker_go_units//:go-units", "@com_github_google_btree//:btree", @@ -37,6 +38,7 @@ go_test( shard_count = 7, deps = [ ":prepare_snap", + "//br/pkg/utils", "//store/mockstore/unistore", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", diff --git a/br/pkg/checkpoint/BUILD.bazel b/br/pkg/checkpoint/BUILD.bazel index 050f63f9650aa..20e39dc39025b 100644 --- a/br/pkg/checkpoint/BUILD.bazel +++ b/br/pkg/checkpoint/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//br/pkg/rtree", "//br/pkg/storage", "//br/pkg/summary", + "//br/pkg/utils", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", "@com_github_pingcap_log//:log", diff --git a/br/pkg/conn/BUILD.bazel b/br/pkg/conn/BUILD.bazel index b05e58a2d879e..fc88f174394f3 100644 --- a/br/pkg/conn/BUILD.bazel +++ b/br/pkg/conn/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//br/pkg/glue", "//br/pkg/logutil", "//br/pkg/pdutil", + "//br/pkg/utils", "//br/pkg/version", "//domain", "//kv", @@ -46,6 +47,7 @@ go_test( deps = [ "//br/pkg/conn/util", "//br/pkg/pdutil", + "//br/pkg/utils", "//testkit/testsetup", "@com_github_docker_go_units//:go-units", "@com_github_pingcap_errors//:errors", diff --git a/br/pkg/glue/BUILD.bazel b/br/pkg/glue/BUILD.bazel index f20f1c0bfc165..dc993f47c31d8 100644 --- a/br/pkg/glue/BUILD.bazel +++ b/br/pkg/glue/BUILD.bazel @@ -10,6 +10,7 @@ go_library( importpath = "github.com/pingcap/tidb/br/pkg/glue", visibility = ["//visibility:public"], deps = [ + "//br/pkg/utils", "//ddl", "//domain", "//kv", diff --git a/br/pkg/gluetikv/BUILD.bazel b/br/pkg/gluetikv/BUILD.bazel index b762691a3a129..2c82b6d85b40f 100644 --- a/br/pkg/gluetikv/BUILD.bazel +++ b/br/pkg/gluetikv/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//br/pkg/glue", "//br/pkg/summary", + "//br/pkg/utils", "//br/pkg/version/build", "//config", "//domain", diff --git a/br/pkg/lightning/BUILD.bazel b/br/pkg/lightning/BUILD.bazel index 7b201540598b1..95aca448f786a 100644 --- a/br/pkg/lightning/BUILD.bazel +++ b/br/pkg/lightning/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "//br/pkg/lightning/web", "//br/pkg/redact", "//br/pkg/storage", + "//br/pkg/utils", "//br/pkg/version/build", "//expression", "//planner/core", diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel index ac216cd0f47c5..1696195d4abe4 100644 --- a/br/pkg/lightning/backend/kv/BUILD.bazel +++ b/br/pkg/lightning/backend/kv/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "//br/pkg/lightning/verification", "//br/pkg/logutil", "//br/pkg/redact", + "//br/pkg/utils", "//expression", "//kv", "//meta/autoid", diff --git a/br/pkg/lightning/backend/local/BUILD.bazel b/br/pkg/lightning/backend/local/BUILD.bazel index 40767b876dccf..c17ad0b92ec1c 100644 --- a/br/pkg/lightning/backend/local/BUILD.bazel +++ b/br/pkg/lightning/backend/local/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "//br/pkg/membuf", "//br/pkg/pdutil", "//br/pkg/restore/split", + "//br/pkg/utils", "//br/pkg/version", "//distsql", "//infoschema", @@ -107,6 +108,7 @@ go_test( "//br/pkg/mock", "//br/pkg/pdutil", "//br/pkg/restore/split", + "//br/pkg/utils", "//ddl", "//kv", "//parser", diff --git a/br/pkg/lightning/backend/tidb/BUILD.bazel b/br/pkg/lightning/backend/tidb/BUILD.bazel index 5e6f3c1a546d2..9dd8a9a876c14 100644 --- a/br/pkg/lightning/backend/tidb/BUILD.bazel +++ b/br/pkg/lightning/backend/tidb/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//br/pkg/lightning/log", "//br/pkg/lightning/verification", "//br/pkg/redact", + "//br/pkg/utils", "//br/pkg/version", "//parser/model", "//parser/mysql", diff --git a/br/pkg/lightning/common/BUILD.bazel b/br/pkg/lightning/common/BUILD.bazel index 9b19f08cebd06..964e01f31d4fa 100644 --- a/br/pkg/lightning/common/BUILD.bazel +++ b/br/pkg/lightning/common/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "//br/pkg/errors", "//br/pkg/httputil", "//br/pkg/lightning/log", + "//br/pkg/utils", "//errno", "//parser/model", "//store/driver/error", diff --git a/br/pkg/lightning/errormanager/BUILD.bazel b/br/pkg/lightning/errormanager/BUILD.bazel index 855c13d6eed41..c557eca4c7ce4 100644 --- a/br/pkg/lightning/errormanager/BUILD.bazel +++ b/br/pkg/lightning/errormanager/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//br/pkg/lightning/config", "//br/pkg/lightning/log", "//br/pkg/redact", + "//br/pkg/utils", "@com_github_jedib0t_go_pretty_v6//table", "@com_github_jedib0t_go_pretty_v6//text", "@com_github_pingcap_errors//:errors", @@ -28,6 +29,7 @@ go_test( deps = [ "//br/pkg/lightning/config", "//br/pkg/lightning/log", + "//br/pkg/utils", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_stretchr_testify//require", "@org_uber_go_atomic//:atomic", diff --git a/br/pkg/lightning/restore/BUILD.bazel b/br/pkg/lightning/restore/BUILD.bazel index c37d3a77b20c6..3c2be48b86e19 100644 --- a/br/pkg/lightning/restore/BUILD.bazel +++ b/br/pkg/lightning/restore/BUILD.bazel @@ -40,6 +40,7 @@ go_library( "//br/pkg/redact", "//br/pkg/storage", "//br/pkg/streamhelper", + "//br/pkg/utils", "//br/pkg/version", "//br/pkg/version/build", "//config", diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel index 8da6cc8bf3f3f..85f0836e28d9f 100644 --- a/br/pkg/restore/BUILD.bazel +++ b/br/pkg/restore/BUILD.bazel @@ -42,6 +42,7 @@ go_library( "//br/pkg/storage", "//br/pkg/stream", "//br/pkg/summary", + "//br/pkg/utils", "//br/pkg/utils/iter", "//br/pkg/utils/storewatch", "//br/pkg/version", @@ -140,6 +141,7 @@ go_test( "//br/pkg/rtree", "//br/pkg/storage", "//br/pkg/stream", + "//br/pkg/utils", "//br/pkg/utils/iter", "//infoschema", "//kv", diff --git a/br/pkg/restore/split/BUILD.bazel b/br/pkg/restore/split/BUILD.bazel index 1ae186e9f22cc..920f8ec302267 100644 --- a/br/pkg/restore/split/BUILD.bazel +++ b/br/pkg/restore/split/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//br/pkg/httputil", "//br/pkg/logutil", "//br/pkg/redact", + "//br/pkg/utils", "//store/pdtypes", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", @@ -42,6 +43,7 @@ go_test( deps = [ ":split", "//br/pkg/errors", + "//br/pkg/utils", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", ], diff --git a/br/pkg/restore/tiflashrec/BUILD.bazel b/br/pkg/restore/tiflashrec/BUILD.bazel index 16ffa1b2564ed..fbe6ba415cb8c 100644 --- a/br/pkg/restore/tiflashrec/BUILD.bazel +++ b/br/pkg/restore/tiflashrec/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//br/pkg/logutil", + "//br/pkg/utils", "//infoschema", "//parser/ast", "//parser/format", diff --git a/br/pkg/stream/BUILD.bazel b/br/pkg/stream/BUILD.bazel index cb3d585dc8242..f75f9f37d81ea 100644 --- a/br/pkg/stream/BUILD.bazel +++ b/br/pkg/stream/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "//br/pkg/logutil", "//br/pkg/storage", "//br/pkg/streamhelper", + "//br/pkg/utils", "//kv", "//meta", "//parser/model", diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel index 098577060045f..63b4dee0d18b0 100644 --- a/br/pkg/streamhelper/BUILD.bazel +++ b/br/pkg/streamhelper/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "//br/pkg/redact", "//br/pkg/streamhelper/config", "//br/pkg/streamhelper/spans", + "//br/pkg/utils", "//config", "//kv", "//metrics", @@ -76,6 +77,7 @@ go_test( "//br/pkg/storage", "//br/pkg/streamhelper/config", "//br/pkg/streamhelper/spans", + "//br/pkg/utils", "//kv", "//tablecodec", "//util/codec", diff --git a/br/pkg/streamhelper/spans/BUILD.bazel b/br/pkg/streamhelper/spans/BUILD.bazel index a8de313029f43..899f6f6ade6b1 100644 --- a/br/pkg/streamhelper/spans/BUILD.bazel +++ b/br/pkg/streamhelper/spans/BUILD.bazel @@ -11,6 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//br/pkg/logutil", + "//br/pkg/utils", "//kv", "@com_github_google_btree//:btree", ], diff --git a/br/pkg/task/BUILD.bazel b/br/pkg/task/BUILD.bazel index 8a336a4e32f40..4b24280e3f4bf 100644 --- a/br/pkg/task/BUILD.bazel +++ b/br/pkg/task/BUILD.bazel @@ -39,6 +39,7 @@ go_library( "//br/pkg/streamhelper/config", "//br/pkg/streamhelper/daemon", "//br/pkg/summary", + "//br/pkg/utils", "//br/pkg/version", "//config", "//ddl", @@ -102,6 +103,7 @@ go_test( "//br/pkg/restore", "//br/pkg/storage", "//br/pkg/stream", + "//br/pkg/utils", "//config", "//parser/model", "//statistics/handle", diff --git a/br/pkg/task/operator/BUILD.bazel b/br/pkg/task/operator/BUILD.bazel index 8d89fb2a40ec1..52c99c845b57b 100644 --- a/br/pkg/task/operator/BUILD.bazel +++ b/br/pkg/task/operator/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//br/pkg/logutil", "//br/pkg/pdutil", "//br/pkg/task", + "//br/pkg/utils", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_log//:log", "@com_github_spf13_pflag//:pflag", diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel index a361838df308e..d8b42c9c42b86 100644 --- a/br/pkg/utils/BUILD.bazel +++ b/br/pkg/utils/BUILD.bazel @@ -89,15 +89,7 @@ go_test( ], embed = [":utils"], flaky = True, -<<<<<<< HEAD -<<<<<<< HEAD - shard_count = 29, -======= - shard_count = 37, ->>>>>>> 6c30c6e6aa7 (br: refactor error handle mechanism to tolerant unexpect kv errors. (#48646)) -======= shard_count = 34, ->>>>>>> release-6.5 deps = [ "//br/pkg/errors", "//br/pkg/metautil", @@ -117,12 +109,7 @@ go_test( "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", "@com_github_pingcap_kvproto//pkg/encryptionpb", -<<<<<<< HEAD "@com_github_pingcap_kvproto//pkg/errorpb", - "@com_github_pingcap_kvproto//pkg/import_sstpb", - "@com_github_pingcap_kvproto//pkg/metapb", -======= ->>>>>>> release-6.5 "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_pd_client//:client", diff --git a/br/pkg/utils/iter/BUILD.bazel b/br/pkg/utils/iter/BUILD.bazel index 4c39e57a0257b..0e4c55ed67d56 100644 --- a/br/pkg/utils/iter/BUILD.bazel +++ b/br/pkg/utils/iter/BUILD.bazel @@ -12,6 +12,7 @@ go_library( importpath = "github.com/pingcap/tidb/br/pkg/utils/iter", visibility = ["//visibility:public"], deps = [ + "//br/pkg/utils", "@org_golang_x_exp//constraints", "@org_golang_x_sync//errgroup", ], diff --git a/br/pkg/utils/retry.go b/br/pkg/utils/retry.go index f928e65be6dd6..572065fefc623 100644 --- a/br/pkg/utils/retry.go +++ b/br/pkg/utils/retry.go @@ -4,6 +4,7 @@ package utils import ( "context" + stderrs "errors" "fmt" "strings" "sync" diff --git a/br/pkg/version/BUILD.bazel b/br/pkg/version/BUILD.bazel index 5d759bc6a404b..f969242b7aeaa 100644 --- a/br/pkg/version/BUILD.bazel +++ b/br/pkg/version/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//br/pkg/errors", "//br/pkg/logutil", + "//br/pkg/utils", "//br/pkg/version/build", "//parser/model", "//sessionctx/variable", diff --git a/ddl/BUILD.bazel b/ddl/BUILD.bazel index 8cc648da21b2f..7072c6cb93558 100644 --- a/ddl/BUILD.bazel +++ b/ddl/BUILD.bazel @@ -54,6 +54,7 @@ go_library( ], deps = [ "//br/pkg/lightning/common", + "//br/pkg/utils", "//config", "//ddl/ingest", "//ddl/label", diff --git a/dumpling/export/BUILD.bazel b/dumpling/export/BUILD.bazel index 274fb6315289b..fc3006cb855c0 100644 --- a/dumpling/export/BUILD.bazel +++ b/dumpling/export/BUILD.bazel @@ -28,6 +28,7 @@ go_library( deps = [ "//br/pkg/storage", "//br/pkg/summary", + "//br/pkg/utils", "//br/pkg/version", "//config", "//dumpling/cli", diff --git a/telemetry/BUILD.bazel b/telemetry/BUILD.bazel index 3d0a0a8e7004e..1f032aa3f237a 100644 --- a/telemetry/BUILD.bazel +++ b/telemetry/BUILD.bazel @@ -18,6 +18,7 @@ go_library( importpath = "github.com/pingcap/tidb/telemetry", visibility = ["//visibility:public"], deps = [ + "//br/pkg/utils", "//config", "//domain/infosync", "//infoschema", From 8542e71b897c710a4cb9ac946e652aaaf612d916 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Mon, 29 Jan 2024 15:33:08 +0800 Subject: [PATCH 4/4] fix build --- br/pkg/utils/retry_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/br/pkg/utils/retry_test.go b/br/pkg/utils/retry_test.go index df0b99e95cacd..f93c8f64a76b6 100644 --- a/br/pkg/utils/retry_test.go +++ b/br/pkg/utils/retry_test.go @@ -11,6 +11,7 @@ import ( "github.com/pingcap/errors" backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/kvproto/pkg/errorpb" + berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/utils" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv"