From 34e04b1053f7cbb3f4048b4e04e5541056e6dfe4 Mon Sep 17 00:00:00 2001 From: Leavrth Date: Sun, 12 Nov 2023 13:39:39 +0800 Subject: [PATCH 1/2] add more retryable error for br_import Signed-off-by: Leavrth --- br/pkg/utils/retry.go | 1 + br/tests/br_full/run.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/br/pkg/utils/retry.go b/br/pkg/utils/retry.go index 130a1402ec149..56d567c09004e 100644 --- a/br/pkg/utils/retry.go +++ b/br/pkg/utils/retry.go @@ -30,6 +30,7 @@ var retryableServerError = []string{ "not read from or written to within the timeout period", "requesttimeout", "invalidpart", + "end of file before message length reached", } // RetryableFunc presents a retryable operation. diff --git a/br/tests/br_full/run.sh b/br/tests/br_full/run.sh index 1753dfdc460d1..e68fb6b39c871 100755 --- a/br/tests/br_full/run.sh +++ b/br/tests/br_full/run.sh @@ -53,7 +53,7 @@ test_log="${TEST_DIR}/${DB}_test.log" error_str="not read from or written to within the timeout period" unset BR_LOG_TO_TERM -export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/backup/backup-storage-error=1*return(\"connection refused\")->1*return(\"InternalError\");github.com/pingcap/tidb/br/pkg/backup/backup-timeout-error=1*return(\"RequestTimeout\")->1*return(\"not read from or written to within the timeout period\")->1*return(\"InvalidPart\")" +export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/backup/backup-storage-error=1*return(\"connection refused\")->1*return(\"InternalError\");github.com/pingcap/tidb/br/pkg/backup/backup-timeout-error=1*return(\"RequestTimeout\")->1*return(\"not read from or written to within the timeout period\")->1*return(\"InvalidPart\")->1*return(\"end of file before message length reached\")" run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB-lz4" --concurrency 4 --compression lz4 --log-file $test_log export GO_FAILPOINTS="" size_lz4=$(du -d 0 $TEST_DIR/$DB-lz4 | awk '{print $1}') From efa0bef51877d7c2ec0b99b83335980973563e0b Mon Sep 17 00:00:00 2001 From: Leavrth Date: Mon, 13 Nov 2023 11:37:12 +0800 Subject: [PATCH 2/2] add log when retry to import ssts Signed-off-by: Leavrth --- br/pkg/utils/backoff.go | 1 + 1 file changed, 1 insertion(+) diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go index 368c11b7f0b6a..8c7712ecb1b1d 100644 --- a/br/pkg/utils/backoff.go +++ b/br/pkg/utils/backoff.go @@ -140,6 +140,7 @@ func NewDownloadSSTBackoffer() Backoffer { } func (bo *importerBackoffer) NextBackoff(err error) time.Duration { + log.Warn("retry to import ssts", zap.Int("attempt", bo.attempt), zap.Error(err)) if MessageIsRetryableStorageError(err.Error()) { bo.delayTime = 2 * bo.delayTime bo.attempt--