Merge branch 'master' of https://github.com/pingcap/tidb into session…

…-rg-name
pingcap · Mar 6, 2023 · cfd6c20 · cfd6c20
2 parents 2281510 + 196ddc0
commit cfd6c20
Show file tree

Hide file tree

Showing 145 changed files with 10,691 additions and 8,477 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -6,6 +6,7 @@ http_archive(
     urls = [
         "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.38.1/rules_go-v0.38.1.zip",
         "https://github.com/bazelbuild/rules_go/releases/download/v0.38.1/rules_go-v0.38.1.zip",
+        "https://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.38.1/rules_go-v0.38.1.zip",
     ],
 )
 
@@ -15,9 +16,20 @@ http_archive(
     urls = [
         "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz",
         "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz",
+        "http://ats.apps.svc/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz",
     ],
 )
 
+http_archive(
+    name = "rules_cc",
+    urls = [
+        "https://github.com/bazelbuild/rules_cc/releases/download/0.0.6/rules_cc-0.0.6.tar.gz",
+        "http://ats.apps.svc/bazelbuild/rules_cc/releases/download/0.0.6/rules_cc-0.0.6.tar.gz",
+    ],
+    sha256 = "3d9e271e2876ba42e114c9b9bc51454e379cbf0ec9ef9d40e2ae4cec61a31b40",
+    strip_prefix = "rules_cc-0.0.6",
+)
+
 load("@io_bazel_rules_go//go:deps.bzl", "go_download_sdk", "go_register_toolchains", "go_rules_dependencies")
 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
 load("//:DEPS.bzl", "go_deps")

diff --git a/br/cmd/br/BUILD.bazel b/br/cmd/br/BUILD.bazel
@@ -34,6 +34,7 @@ go_library(
         "//session",
         "//util",
         "//util/logutil",
+        "//util/metricsutil",
         "@com_github_gogo_protobuf//proto",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_kvproto//pkg/brpb",

diff --git a/br/cmd/br/backup.go b/br/cmd/br/backup.go
@@ -13,6 +13,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/version/build"
 	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/session"
+	"github.com/pingcap/tidb/util/metricsutil"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 	"sourcegraph.com/sourcegraph/appdash"
@@ -25,6 +26,10 @@ func runBackupCommand(command *cobra.Command, cmdName string) error {
 		return errors.Trace(err)
 	}
 
+	if err := metricsutil.RegisterMetricsForBR(cfg.PD, cfg.KeyspaceName); err != nil {
+		return errors.Trace(err)
+	}
+
 	ctx := GetDefaultContext()
 	if cfg.EnableOpenTracing {
 		var store *appdash.MemoryStore

diff --git a/br/cmd/br/restore.go b/br/cmd/br/restore.go
@@ -15,6 +15,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/utils"
 	"github.com/pingcap/tidb/br/pkg/version/build"
 	"github.com/pingcap/tidb/session"
+	"github.com/pingcap/tidb/util/metricsutil"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 	"sourcegraph.com/sourcegraph/appdash"
@@ -27,6 +28,10 @@ func runRestoreCommand(command *cobra.Command, cmdName string) error {
 		return errors.Trace(err)
 	}
 
+	if err := metricsutil.RegisterMetricsForBR(cfg.PD, cfg.KeyspaceName); err != nil {
+		return errors.Trace(err)
+	}
+
 	if task.IsStreamRestore(cmdName) {
 		if err := cfg.ParseStreamRestoreFlags(command.Flags()); err != nil {
 			return errors.Trace(err)

diff --git a/br/pkg/lightning/backend/kv/sql2kv.go b/br/pkg/lightning/backend/kv/sql2kv.go
@@ -481,7 +481,7 @@ func (kvcodec *tableKVEncoder) getActualDatum(rowID int64, colIndex int, inputDa
 		if err != nil {
 			return value, err
 		}
-		if err := col.CheckNotNull(&value); err == nil {
+		if err := col.CheckNotNull(&value, 0); err == nil {
 			return value, nil // the most normal case
 		}
 		isBadNullValue = true
@@ -505,7 +505,7 @@ func (kvcodec *tableKVEncoder) getActualDatum(rowID int64, colIndex int, inputDa
 		// if MutRowFromDatums sees a nil it won't initialize the underlying storage and cause SetDatum to panic.
 		value = types.GetMinValue(&col.FieldType)
 	case isBadNullValue:
-		err = col.HandleBadNull(&value, kvcodec.se.vars.StmtCtx)
+		err = col.HandleBadNull(&value, kvcodec.se.vars.StmtCtx, 0)
 	default:
 		value, err = table.GetColDefaultValue(kvcodec.se, col.ToInfo())
 	}

diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go
@@ -218,7 +218,7 @@ func (t PostOpLevel) MarshalText() ([]byte, error) {
 	return []byte(t.String()), nil
 }
 
-// parser command line parameter
+// FromStringValue parse command line parameter.
 func (t *PostOpLevel) FromStringValue(s string) error {
 	switch strings.ToLower(s) {
 	//nolint:goconst // This 'false' and other 'false's aren't the same.

diff --git a/br/pkg/lightning/restore/get_pre_info.go b/br/pkg/lightning/restore/get_pre_info.go
@@ -398,6 +398,7 @@ get_struct_from_src:
 			return nil, errors.Annotatef(err, "get create table statement from schema file error: %s", tableFileMeta.Name)
 		}
 		theTableInfo, err := newTableInfo(createTblSQL, 0)
+		log.L().Info("generate table info from SQL", zap.Error(err), zap.String("sql", createTblSQL), zap.String("table_name", tableFileMeta.Name), zap.String("db_name", dbSrcFileMeta.Name))
 		if err != nil {
 			errMsg := "generate table info from SQL error"
 			log.L().Error(errMsg, zap.Error(err), zap.String("sql", createTblSQL), zap.String("table_name", tableFileMeta.Name))

diff --git a/br/pkg/lightning/restore/precheck_impl.go b/br/pkg/lightning/restore/precheck_impl.go
@@ -590,9 +590,13 @@ func (ci *checkpointCheckItem) Check(ctx context.Context) (*CheckResult, error)
 	}
 
 	checkMsgs := []string{}
+	dbInfos, err := ci.preInfoGetter.GetAllTableStructures(ctx)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
 	for _, dbInfo := range ci.dbMetas {
 		for _, tableInfo := range dbInfo.Tables {
-			msgs, err := ci.checkpointIsValid(ctx, tableInfo)
+			msgs, err := ci.checkpointIsValid(ctx, tableInfo, dbInfos)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
@@ -607,7 +611,7 @@ func (ci *checkpointCheckItem) Check(ctx context.Context) (*CheckResult, error)
 }
 
 // checkpointIsValid checks whether we can start this import with this checkpoint.
-func (ci *checkpointCheckItem) checkpointIsValid(ctx context.Context, tableInfo *mydump.MDTableMeta) ([]string, error) {
+func (ci *checkpointCheckItem) checkpointIsValid(ctx context.Context, tableInfo *mydump.MDTableMeta, dbInfos map[string]*checkpoints.TidbDBInfo) ([]string, error) {
 	msgs := make([]string, 0)
 	uniqueName := common.UniqueTable(tableInfo.DB, tableInfo.Name)
 	tableCheckPoint, err := ci.checkpointsDB.Get(ctx, uniqueName)
@@ -646,10 +650,6 @@ func (ci *checkpointCheckItem) checkpointIsValid(ctx context.Context, tableInfo
 		return msgs, nil
 	}
 
-	dbInfos, err := ci.preInfoGetter.GetAllTableStructures(ctx)
-	if err != nil {
-		return nil, errors.Trace(err)
-	}
 	dbInfo, ok := dbInfos[tableInfo.DB]
 	if ok {
 		t, ok := dbInfo.Tables[tableInfo.Name]
@@ -850,6 +850,11 @@ func (ci *schemaCheckItem) Check(ctx context.Context) (*CheckResult, error) {
 		Message:  "table schemas are valid",
 	}
 
+	dbInfos, err := ci.preInfoGetter.GetAllTableStructures(ctx)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
 	checkMsgs := []string{}
 	for _, dbInfo := range ci.dbMetas {
 		for _, tableInfo := range dbInfo.Tables {
@@ -861,7 +866,7 @@ func (ci *schemaCheckItem) Check(ctx context.Context) (*CheckResult, error) {
 					continue
 				}
 			}
-			msgs, err := ci.SchemaIsValid(ctx, tableInfo)
+			msgs, err := ci.SchemaIsValid(ctx, tableInfo, dbInfos)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
@@ -876,17 +881,14 @@ func (ci *schemaCheckItem) Check(ctx context.Context) (*CheckResult, error) {
 }
 
 // SchemaIsValid checks the import file and cluster schema is match.
-func (ci *schemaCheckItem) SchemaIsValid(ctx context.Context, tableInfo *mydump.MDTableMeta) ([]string, error) {
+func (ci *schemaCheckItem) SchemaIsValid(ctx context.Context, tableInfo *mydump.MDTableMeta, dbInfos map[string]*checkpoints.TidbDBInfo) ([]string, error) {
 	if len(tableInfo.DataFiles) == 0 {
 		log.FromContext(ctx).Info("no data files detected", zap.String("db", tableInfo.DB), zap.String("table", tableInfo.Name))
 		return nil, nil
 	}
 
 	msgs := make([]string, 0)
-	dbInfos, err := ci.preInfoGetter.GetAllTableStructures(ctx)
-	if err != nil {
-		return nil, errors.Trace(err)
-	}
+
 	info, ok := dbInfos[tableInfo.DB].Tables[tableInfo.Name]
 	if !ok {
 		msgs = append(msgs, fmt.Sprintf("TiDB schema `%s`.`%s` doesn't exists,"+

diff --git a/br/pkg/lightning/restore/table_restore_test.go b/br/pkg/lightning/restore/table_restore_test.go
@@ -2157,7 +2157,7 @@ func (s *tableRestoreSuite) TestSchemaIsValid() {
 		}
 		ci := NewSchemaCheckItem(cfg, preInfoGetter, nil, nil).(*schemaCheckItem)
 		preInfoGetter.dbInfosCache = ca.dbInfos
-		msgs, err := ci.SchemaIsValid(ctx, ca.tableMeta)
+		msgs, err := ci.SchemaIsValid(ctx, ca.tableMeta, ca.dbInfos)
 		require.NoError(s.T(), err)
 		require.Len(s.T(), msgs, ca.MsgNum)
 		if len(msgs) > 0 {
@@ -2241,7 +2241,7 @@ func (s *tableRestoreSuite) TestGBKEncodedSchemaIsValid() {
 				},
 			},
 		},
-	})
+	}, dbInfos)
 	require.NoError(s.T(), err)
 	require.Len(s.T(), msgs, 0)
 }