Merge remote-tracking branch 'upstream/master' into issue52207

.github/licenserc.yml

@@ -54,4 +54,13 @@ header:
     - "pkg/disttask/**/mock/**/*_mock.go"
     - "pkg/util/sqlexec/mock/*_mock.go"
     - "pkg/extension/enterprise/"
+    - "lightning/pkg/web/res_vfsdata.go"
+    - "lightning/web/docs/api.yaml"
+    - "lightning/web/public/index.html"
+    - "lightning/web/webpack.config.js"
+    - "pkg/lightning/checkpoints/checkpointspb/file_checkpoints.pb.go"
+    - "pkg/lightning/manual/manual.go"
+    - "pkg/lightning/manual/manual_nocgo.go"
+    - "pkg/lightning/mydump/bytes.go"
+    - "pkg/lightning/mydump/examples/metadata"
   comment: on-failure

Makefile

@@ -64,7 +64,7 @@ errdoc:tools/bin/errdoc-gen
 lint:tools/bin/revive
 	@echo "linting"
 	@tools/bin/revive -formatter friendly -config tools/check/revive.toml $(FILES_TIDB_TESTS)
-	@tools/bin/revive -formatter friendly -config tools/check/revive.toml ./br/pkg/lightning/...
+	@tools/bin/revive -formatter friendly -config tools/check/revive.toml ./lightning/...
 	go run tools/dashboard-linter/main.go pkg/metrics/grafana/overview.json
 	go run tools/dashboard-linter/main.go pkg/metrics/grafana/performance_overview.json
 	go run tools/dashboard-linter/main.go pkg/metrics/grafana/tidb.json
@@ -314,20 +314,20 @@ bench-daily:
 
 build_tools: build_br build_lightning build_lightning-ctl
 
-br_web:
-	@cd br/web && npm install && npm run build
+lightning_web:
+	@cd lightning/web && npm install && npm run build
 
 build_br:
 	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(BR_BIN) ./br/cmd/br
 
 build_lightning_for_web:
-	CGO_ENABLED=1 $(GOBUILD) -tags dev $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_BIN) br/cmd/tidb-lightning/main.go
+	CGO_ENABLED=1 $(GOBUILD) -tags dev $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_BIN) lightning/cmd/tidb-lightning/main.go
 
 build_lightning:
-	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_BIN) ./br/cmd/tidb-lightning
+	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_BIN) ./lightning/cmd/tidb-lightning
 
 build_lightning-ctl:
-	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_CTL_BIN) ./br/cmd/tidb-lightning-ctl
+	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_CTL_BIN) ./lightning/cmd/tidb-lightning-ctl
 
 build_for_br_integration_test:
 	@make failpoint-enable
@@ -336,13 +336,13 @@ build_for_br_integration_test:
 		-o $(BR_BIN).test \
 		github.com/pingcap/tidb/br/cmd/br && \
 	$(GOTEST) -c -cover -covermode=count \
-		-coverpkg=github.com/pingcap/tidb/br/... \
+		-coverpkg=github.com/pingcap/tidb/lightning/...,github.com/pingcap/tidb/pkg/lightning/... \
 		-o $(LIGHTNING_BIN).test \
-		github.com/pingcap/tidb/br/cmd/tidb-lightning && \
+		github.com/pingcap/tidb/lightning/cmd/tidb-lightning && \
 	$(GOTEST) -c -cover -covermode=count \
-		-coverpkg=github.com/pingcap/tidb/br/... \
+		-coverpkg=github.com/pingcap/tidb/lightning/...,github.com/pingcap/tidb/pkg/lightning/... \
 		-o $(LIGHTNING_CTL_BIN).test \
-		github.com/pingcap/tidb/br/cmd/tidb-lightning-ctl && \
+		github.com/pingcap/tidb/lightning/cmd/tidb-lightning-ctl && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/locker br/tests/br_key_locked/*.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/gc br/tests/br_z_gc_safepoint/*.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/oauth br/tests/br_gcs/*.go && \
@@ -384,9 +384,9 @@ mock_s3iface: mockgen
 
 # mock interface for lightning and IMPORT INTO
 mock_lightning: mockgen
-	tools/bin/mockgen -package mock github.com/pingcap/tidb/br/pkg/lightning/backend Backend,EngineWriter,TargetInfoGetter,ChunkFlushStatus > br/pkg/mock/backend.go
-	tools/bin/mockgen -package mock github.com/pingcap/tidb/br/pkg/lightning/backend/encode Encoder,EncodingBuilder,Rows,Row > br/pkg/mock/encode.go
-	tools/bin/mockgen -package mocklocal github.com/pingcap/tidb/br/pkg/lightning/backend/local DiskUsage,TiKVModeSwitcher,StoreHelper > br/pkg/mock/mocklocal/local.go
+	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/lightning/backend Backend,EngineWriter,TargetInfoGetter,ChunkFlushStatus > br/pkg/mock/backend.go
+	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/lightning/backend/encode Encoder,EncodingBuilder,Rows,Row > br/pkg/mock/encode.go
+	tools/bin/mockgen -package mocklocal github.com/pingcap/tidb/pkg/lightning/backend/local DiskUsage,TiKVModeSwitcher,StoreHelper > br/pkg/mock/mocklocal/local.go
 	tools/bin/mockgen -package mock github.com/pingcap/tidb/br/pkg/utils TaskRegister > br/pkg/mock/task_register.go
 
 gen_mock: mockgen
@@ -426,9 +426,9 @@ br_bins:
 	@echo '// Code generated by ragel DO NOT EDIT.' | cat - tmp_parser.go | sed 's|//line |//.... |g' > $@
 	@rm tmp_parser.go
 
-data_parsers: tools/bin/vfsgendev br/pkg/lightning/mydump/parser_generated.go br_web
-	PATH="$(GOPATH)/bin":"$(PATH)":"$(TOOLS)" protoc -I. -I"$(GOPATH)/src" br/pkg/lightning/checkpoints/checkpointspb/file_checkpoints.proto --gogofaster_out=.
-	tools/bin/vfsgendev -source='"github.com/pingcap/tidb/br/pkg/lightning/web".Res' && mv res_vfsdata.go br/pkg/lightning/web/
+data_parsers: tools/bin/vfsgendev pkg/lightning/mydump/parser_generated.go lightning_web
+	PATH="$(GOPATH)/bin":"$(PATH)":"$(TOOLS)" protoc -I. -I"$(GOPATH)/src" pkg/lightning/checkpoints/checkpointspb/file_checkpoints.proto --gogofaster_out=.
+	tools/bin/vfsgendev -source='"github.com/pingcap/tidb/lightning/pkg/web".Res' && mv res_vfsdata.go lightning/pkg/web/
 
 build_dumpling:
 	$(DUMPLING_GOBUILD) $(RACE_FLAG) -tags codes -o $(DUMPLING_BIN) dumpling/cmd/dumpling/main.go

WORKSPACE

@@ -64,7 +64,7 @@ go_download_sdk(
         "https://mirrors.aliyun.com/golang/{}",
         "https://dl.google.com/go/{}",
     ],
-    version = "1.21.6",
+    version = "1.21.8",
 )
 
 go_register_toolchains(

br/pkg/logutil/BUILD.bazel

@@ -10,8 +10,8 @@ go_library(
     importpath = "github.com/pingcap/tidb/br/pkg/logutil",
     visibility = ["//visibility:public"],
     deps = [
-        "//br/pkg/lightning/metric",
         "//pkg/kv",
+        "//pkg/lightning/metric",
         "//pkg/util/redact",
         "@com_github_google_uuid//:uuid",
         "@com_github_pingcap_errors//:errors",

br/pkg/logutil/rate.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/pingcap/log"
-	"github.com/pingcap/tidb/br/pkg/lightning/metric"
+	"github.com/pingcap/tidb/pkg/lightning/metric"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 )

br/pkg/mock/BUILD.bazel

@@ -13,12 +13,12 @@ go_library(
     importpath = "github.com/pingcap/tidb/br/pkg/mock",
     visibility = ["//visibility:public"],
     deps = [
-        "//br/pkg/lightning/backend",
-        "//br/pkg/lightning/backend/encode",
-        "//br/pkg/lightning/verification",
         "//pkg/config",
         "//pkg/domain",
         "//pkg/kv",
+        "//pkg/lightning/backend",
+        "//pkg/lightning/backend/encode",
+        "//pkg/lightning/verification",
         "//pkg/parser/model",
         "//pkg/server",
         "//pkg/session",

br/pkg/mock/mocklocal/BUILD.bazel

@@ -6,7 +6,7 @@ go_library(
     importpath = "github.com/pingcap/tidb/br/pkg/mock/mocklocal",
     visibility = ["//visibility:public"],
     deps = [
-        "//br/pkg/lightning/backend",
+        "//pkg/lightning/backend",
         "@com_github_pingcap_kvproto//pkg/import_sstpb",
         "@com_github_tikv_client_go_v2//tikv",
         "@org_uber_go_mock//gomock",

br/pkg/restore/split/BUILD.bazel

@@ -14,12 +14,12 @@ go_library(
     deps = [
         "//br/pkg/conn/util",
         "//br/pkg/errors",
-        "//br/pkg/lightning/common",
-        "//br/pkg/lightning/config",
-        "//br/pkg/lightning/log",
         "//br/pkg/logutil",
         "//br/pkg/utils",
         "//pkg/kv",
+        "//pkg/lightning/common",
+        "//pkg/lightning/config",
+        "//pkg/lightning/log",
         "//pkg/store/pdtypes",
         "//pkg/util/codec",
         "//pkg/util/intest",

br/pkg/restore/split/client.go

@@ -22,11 +22,11 @@ import (
 	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/br/pkg/conn/util"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
-	"github.com/pingcap/tidb/br/pkg/lightning/common"
-	"github.com/pingcap/tidb/br/pkg/lightning/config"
-	brlog "github.com/pingcap/tidb/br/pkg/lightning/log"
 	"github.com/pingcap/tidb/br/pkg/logutil"
 	"github.com/pingcap/tidb/br/pkg/utils"
+	"github.com/pingcap/tidb/pkg/lightning/common"
+	"github.com/pingcap/tidb/pkg/lightning/config"
+	brlog "github.com/pingcap/tidb/pkg/lightning/log"
 	"github.com/pingcap/tidb/pkg/util/intest"
 	pd "github.com/tikv/pd/client"
 	pdhttp "github.com/tikv/pd/client/http"

br/pkg/restore/split/split.go

@@ -12,9 +12,9 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/log"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
-	"github.com/pingcap/tidb/br/pkg/lightning/config"
 	"github.com/pingcap/tidb/br/pkg/logutil"
 	"github.com/pingcap/tidb/br/pkg/utils"
+	"github.com/pingcap/tidb/pkg/lightning/config"
 	"github.com/pingcap/tidb/pkg/util/codec"
 	"github.com/pingcap/tidb/pkg/util/redact"
 	"go.uber.org/zap"

br/pkg/storage/BUILD.bazel

@@ -26,8 +26,8 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//br/pkg/errors",
-        "//br/pkg/lightning/log",
         "//br/pkg/logutil",
+        "//pkg/lightning/log",
         "//pkg/sessionctx/variable",
         "//pkg/util",
         "//pkg/util/intest",

br/pkg/storage/storage.go

@@ -11,7 +11,7 @@ import (
 	"github.com/pingcap/errors"
 	backuppb "github.com/pingcap/kvproto/pkg/brpb"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
-	"github.com/pingcap/tidb/br/pkg/lightning/log"
+	"github.com/pingcap/tidb/pkg/lightning/log"
 	"go.uber.org/zap"
 )
 

br/pkg/streamhelper/advancer_test.go

@@ -265,21 +265,20 @@ func TestClearCache(t *testing.T) {
 		return nil
 	}
 	failedStoreID := uint64(0)
-	hasFailed := false
+	hasFailed := atomic.NewBool(false)
 	for _, s := range c.stores {
 		s.clientMu.Lock()
+		sid := s.GetID()
 		s.onGetRegionCheckpoint = func(glftrr *logbackup.GetLastFlushTSOfRegionRequest) error {
-			// mark this store cache cleared
-			failedStoreID = s.GetID()
-			if !hasFailed {
-				hasFailed = true
+			// mark one store failed is enough
+			if hasFailed.CompareAndSwap(false, true) {
+				// mark this store cache cleared
+				failedStoreID = sid
 				return errors.New("failed to get checkpoint")
 			}
 			return nil
 		}
 		s.clientMu.Unlock()
-		// mark one store failed is enough
-		break
 	}
 	env := &testEnv{fakeCluster: c, testCtx: t}
 	adv := streamhelper.NewCheckpointAdvancer(env)

br/tests/lightning_add_index/run.sh

@@ -17,7 +17,7 @@
 set -eu
 
 CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/importer/AddIndexFail=return()"
+export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/AddIndexFail=return()"
 
 LOG_FILE1="$TEST_DIR/lightning-add-index1.log"
 LOG_FILE2="$TEST_DIR/lightning-add-index2.log"
@@ -82,7 +82,7 @@ grep -Fq "ALTER TABLE \`add_index\`.\`non_pk_auto_inc\` DROP PRIMARY KEY" "$LOG_
 grep -Fq "ALTER TABLE \`add_index\`.\`non_pk_auto_inc\` ADD PRIMARY KEY (\`pk\`)" "$LOG_FILE2"
 
 # 3. Check for recovering from checkpoint
-export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/importer/AddIndexCrash=return()"
+export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/AddIndexCrash=return()"
 run_sql "DROP DATABASE add_index;"
 run_lightning --enable-checkpoint=1 --config "$CUR/config2.toml" --log-file "$LOG_FILE2"
 grep -Fq "task canceled" "$LOG_FILE2"

br/tests/lightning_check_partial_imported/run.sh

@@ -19,7 +19,7 @@ set -eux
 
 check_cluster_version 4 0 0 'local backend' || exit 0
 
-export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/importer/FailBeforeStartImportingIndexEngine=return"
+export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/FailBeforeStartImportingIndexEngine=return"
 set +e
 if run_lightning; then
     echo "The first import doesn't fail as expected" >&2

br/tests/lightning_checkpoint/run.sh

@@ -62,8 +62,8 @@ check_cluster_version 4 0 0 'local backend'
 
 # Set the failpoint to kill the lightning instance as soon as one table is imported
 # If checkpoint does work, this should only kill 9 instances of lightnings.
-SLOWDOWN_FAILPOINTS='github.com/pingcap/tidb/br/pkg/lightning/importer/SlowDownImport=sleep(250)'
-export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb/br/pkg/lightning/importer/FailBeforeIndexEngineImported=return"
+SLOWDOWN_FAILPOINTS='github.com/pingcap/tidb/lightning/pkg/importer/SlowDownImport=sleep(250)'
+export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb/lightning/pkg/importer/FailBeforeIndexEngineImported=return"
 
 # Start importing the tables.
 run_sql 'DROP DATABASE IF EXISTS cppk_tsr'
@@ -97,7 +97,7 @@ run_sql 'DROP DATABASE IF EXISTS cppk_tsr'
 run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk'
 run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680.bak`'
 
-export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb/br/pkg/lightning/SetTaskID=return(1357924680);github.com/pingcap/tidb/br/pkg/lightning/importer/FailIfIndexEngineImported=return(1)"
+export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb/lightning/pkg/server/SetTaskID=return(1357924680);github.com/pingcap/tidb/lightning/pkg/importer/FailIfIndexEngineImported=return(1)"
 
 set +e
 for i in $(seq "$TABLE_COUNT"); do

br/tests/lightning_checkpoint_chunks/run.sh

@@ -49,16 +49,16 @@ for i in $(seq "$CHUNK_COUNT"); do
     done
 done
 
-PKG="github.com/pingcap/tidb/br/pkg/lightning"
-export GO_FAILPOINTS="$PKG/backend/local/orphanWriterGoRoutine=return();$PKG/restore/orphanWriterGoRoutine=return();$PKG/orphanWriterGoRoutine=return()"
+PKG="github.com/pingcap/tidb/lightning/pkg"
+export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/orphanWriterGoRoutine=return();$PKG/importer/orphanWriterGoRoutine=return();$PKG/server/orphanWriterGoRoutine=return()"
 # test won't panic
 do_run_lightning config
 
 # Set the failpoint to kill the lightning instance as soon as
 # one file (after writing totally $ROW_COUNT rows) is imported.
 # If checkpoint does work, this should kill exactly $CHUNK_COUNT instances of lightnings.
-TASKID_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/SetTaskID=return(1234567890)"
-export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb/br/pkg/lightning/importer/FailIfImportedChunk=return"
+TASKID_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/server/SetTaskID=return(1234567890)"
+export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb/lightning/pkg/importer/FailIfImportedChunk=return"
 
 # Start importing the tables.
 run_sql 'DROP DATABASE IF EXISTS cpch_tsr'
@@ -82,7 +82,7 @@ run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cpch.1234567890
 
 # Set the failpoint to kill the lightning instance as soon as one chunk is imported, via signal mechanism
 # If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
-export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb/br/pkg/lightning/importer/KillIfImportedChunk=return"
+export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb/lightning/pkg/importer/KillIfImportedChunk=return"
 
 for i in $(seq "$CHUNK_COUNT"); do
     echo "******** Importing Chunk Now (step $i/$CHUNK_COUNT) ********"
@@ -98,7 +98,7 @@ rm -f "$TEST_DIR"/cpch.pb*
 
 # Set the failpoint to kill the lightning instance as soon as one chunk is imported
 # If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
-export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb/br/pkg/lightning/importer/FailIfImportedChunk=return"
+export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb/lightning/pkg/importer/FailIfImportedChunk=return"
 set +e
 for i in $(seq "$CHUNK_COUNT"); do
     echo "******** Importing Chunk using File checkpoint Now (step $i/$CHUNK_COUNT) ********"