Merge remote-tracking branch 'upstream/master' into remove-hidden-sys…

…vars

* upstream/master: (28 commits)
  *: enable unconvert (pingcap#35821)
  log-backup: check timezone when using datatime format ts string (pingcap#35811)
  planner: support the leading hint for outer join (pingcap#35264)
  lightning: revert new policy of allocating rowid and refine it later (pingcap#35817)
  txn: manage the initialization of RCCheckTS by transaction context provider (pingcap#35554)
  ddl: make output field name in `show tables/databases stmt` compatible with mysql (pingcap#35136)
  txn: remove `legacy.SimpleTxnContextProvider` (pingcap#35667)
  txn: fix the error message if retry could not be used for rc check ts (pingcap#35474)
  *: support canceling DDL statements with KILL (pingcap#35803)
  lightning: avoid extracting db schema when schema file is not provided (pingcap#35783)
  *: enable all gosimples (pingcap#35762)
  tools/check: skip 'tests' directory in 'make ut' (pingcap#35800)
  metrics: add loading region cache duration (pingcap#34679)
  ddl: DROP TABLE/VIEW/SEQUENCE now use XXXStmt as parameter (pingcap#35741)
  types: fix incompatible implementation of jsonpath extraction (pingcap#35320)
  planner: fix TRACE PLAN TARGET = 'estimation' panic when meeting partition table (pingcap#35743)
  *: Add `testfork.RunTest` to run multiple tests in one function (pingcap#35746)
  sessionctx/variable: add tests to ensure skipInit can be removed (pingcap#35703)
  helper: request another PD if one of them is unavailable (pingcap#35750)
  metrics: add cached table related metrics to grafana panel (pingcap#34718)
  ...

DEPS.bzl

@@ -942,6 +942,13 @@ def go_deps():
         sum = "h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=",
         version = "v0.0.0-20180630174525-215b22d4de21",
     )
+    go_repository(
+        name = "com_github_golangci_unconvert",
+        build_file_proto_mode = "disable",
+        importpath = "github.com/golangci/unconvert",
+        sum = "h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=",
+        version = "v0.0.0-20180507085042-28b1c447d1f4",
+    )
 
     go_repository(
         name = "com_github_gomodule_redigo",

Makefile

@@ -327,14 +327,6 @@ build_for_br_integration_test:
 	) || (make failpoint-disable && exit 1)
 	@make failpoint-disable
 
-build_for_lightning_test:
-	@make failpoint-enable
-	$(GOTEST) -c -cover -covermode=count \
-		-coverpkg=github.com/pingcap/tidb/br/... \
-		-o $(LIGHTNING_BIN).test \
-		github.com/pingcap/tidb/br/cmd/tidb-lightning
-	@make failpoint-disable
-
 br_unit_test: export ARGS=$$($(BR_PACKAGES))
 br_unit_test:
 	@make failpoint-enable

README.md

@@ -62,10 +62,6 @@ The [community repository](https://github.com/pingcap/community) hosts all infor
 
 Contributions are welcomed and greatly appreciated. All the contributors are welcomed to claim your reward by filing this [form](https://forms.pingcap.com/f/tidb-contribution-swag). See [Contribution to TiDB](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/introduction.html) for details on typical contribution workflows. For more contributing information, click on the contributor icon above.
 
-## Adopters
-
-View the current list of in-production TiDB adopters [here](https://docs.pingcap.com/tidb/stable/adopters).
-
 ## Case studies
 
 - [English](https://pingcap.com/case-studies)

br/pkg/lightning/backend/kv/sql2kv.go

@@ -450,49 +450,6 @@ func isPKCol(colInfo *model.ColumnInfo) bool {
 	return mysql.HasPriKeyFlag(colInfo.GetFlag())
 }
 
-func isRowIDOverflow(meta *model.ColumnInfo, rowID int64) bool {
-	isUnsigned := mysql.HasUnsignedFlag(meta.GetFlag())
-	switch meta.GetType() {
-	// MEDIUM INT
-	case mysql.TypeInt24:
-		if !isUnsigned {
-			return rowID > mysql.MaxInt24
-		}
-		return rowID > mysql.MaxUint24
-	// INT
-	case mysql.TypeLong:
-		if !isUnsigned {
-			return rowID > math.MaxInt32
-		}
-		return rowID > math.MaxUint32
-	// SMALLINT
-	case mysql.TypeShort:
-		if !isUnsigned {
-			return rowID > math.MaxInt16
-		}
-		return rowID > math.MaxUint16
-	// TINYINT
-	case mysql.TypeTiny:
-		if !isUnsigned {
-			return rowID > math.MaxInt8
-		}
-		return rowID > math.MaxUint8
-	// FLOAT
-	case mysql.TypeFloat:
-		if !isUnsigned {
-			return float32(rowID) > math.MaxFloat32
-		}
-		return float64(rowID) > math.MaxFloat32*2
-	// DOUBLE
-	case mysql.TypeDouble:
-		if !isUnsigned {
-			return float64(rowID) > math.MaxFloat64
-		}
-		// impossible for rowID exceeding MaxFloat64
-	}
-	return false
-}
-
 func (kvcodec *tableKVEncoder) getActualDatum(rowID int64, colIndex int, inputDatum *types.Datum) (types.Datum, error) {
 	var (
 		value types.Datum
@@ -520,11 +477,6 @@ func (kvcodec *tableKVEncoder) getActualDatum(rowID int64, colIndex int, inputDa
 	// handle special values
 	switch {
 	case isAutoIncCol(col.ToInfo()):
-		// rowID is going to auto-filled the omitted column,
-		// which should be checked before restore
-		if isRowIDOverflow(col.ToInfo(), rowID) {
-			return value, errors.Errorf("PK %d is out of range", rowID)
-		}
 		// we still need a conversion, e.g. to catch overflow with a TINYINT column.
 		value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo(), false, false)
 	case isTableAutoRandom(tblMeta) && isPKCol(col.ToInfo()):

br/pkg/lightning/backend/local/engine.go

@@ -33,7 +33,6 @@ import (
 	"github.com/google/btree"
 	"github.com/google/uuid"
 	"github.com/pingcap/errors"
-	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/br/pkg/lightning/backend"
 	"github.com/pingcap/tidb/br/pkg/lightning/backend/kv"
 	"github.com/pingcap/tidb/br/pkg/lightning/checkpoints"
@@ -1003,21 +1002,6 @@ type Writer struct {
 	batchSize  int64
 
 	lastMetaSeq int32
-	prevRowID   int64 // only used for appendRowsSorted
-}
-
-func (w *Writer) flushAndNewWriter() error {
-	var err error
-	err = w.flush(context.Background())
-	if err != nil {
-		return errors.Trace(err)
-	}
-	newWriter, err := w.createSSTWriter()
-	if err != nil {
-		return errors.Trace(err)
-	}
-	w.writer = newWriter
-	return nil
 }
 
 func (w *Writer) appendRowsSorted(kvs []common.KvPair) error {
@@ -1028,17 +1012,6 @@ func (w *Writer) appendRowsSorted(kvs []common.KvPair) error {
 		}
 		w.writer = writer
 	}
-	if len(kvs) == 0 {
-		return nil
-	}
-	if w.prevRowID != 0 && kvs[0].RowID > w.prevRowID+1 {
-		// rowID leap. probably re-alloc id
-		// should write to different sst
-		err := w.flushAndNewWriter()
-		if err != nil {
-			return err
-		}
-	}
 
 	keyAdapter := w.engine.keyAdapter
 	totalKeySize := 0
@@ -1063,26 +1036,7 @@ func (w *Writer) appendRowsSorted(kvs []common.KvPair) error {
 		}
 		kvs = newKvs
 	}
-	startIdx := 0
-	w.prevRowID = kvs[len(kvs)-1].RowID
-	for i := 1; i < len(kvs); i++ {
-		if kvs[i].RowID > kvs[i-1].RowID+1 {
-			// leap id
-			err := w.writer.writeKVs(kvs[startIdx:i])
-			if err != nil {
-				return err
-			}
-			err = w.flushAndNewWriter()
-			if err != nil {
-				return err
-			}
-			startIdx = i
-		}
-	}
-	if startIdx < len(kvs) {
-		return w.writer.writeKVs(kvs[startIdx:])
-	}
-	return nil
+	return w.writer.writeKVs(kvs)
 }
 
 func (w *Writer) appendRowsUnsorted(ctx context.Context, kvs []common.KvPair) error {
@@ -1149,9 +1103,6 @@ func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames [
 }
 
 func (w *Writer) flush(ctx context.Context) error {
-	failpoint.Inject("MockFlushWriter", func() {
-		failpoint.Return(nil)
-	})
 	w.Lock()
 	defer w.Unlock()
 	if w.batchCount == 0 {

br/pkg/lightning/backend/local/engine_test.go

@@ -29,9 +29,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	"github.com/stretchr/testify/require"
 
-	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/br/pkg/lightning/backend"
-	"github.com/pingcap/tidb/br/pkg/lightning/common"
 )
 
 func TestIngestSSTWithClosedEngine(t *testing.T) {
@@ -87,112 +85,3 @@ func TestIngestSSTWithClosedEngine(t *testing.T) {
 		},
 	}), errorEngineClosed)
 }
-
-func TestAutoSplitSST(t *testing.T) {
-	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/MockFlushWriter", "return(true)"))
-	defer func() {
-		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/MockFlushWriter"))
-	}()
-	var err error
-	dir := os.TempDir()
-	w := &Writer{
-		engine: &Engine{
-			sstDir:     dir,
-			keyAdapter: noopKeyAdapter{},
-			logger:     log.L(),
-		},
-		isKVSorted:         true,
-		isWriteBatchSorted: true,
-	}
-	w.engine.closed.Store(false)
-	w.writer, err = w.createSSTWriter()
-	require.Nil(t, err)
-	kvs := []common.KvPair{
-		{
-			Key:   []byte("1"),
-			Val:   []byte("val1"),
-			RowID: 1,
-		},
-		{
-			Key:   []byte("2"),
-			Val:   []byte("val1"),
-			RowID: 2,
-		},
-	}
-	prevWriter := w.writer
-	err = w.appendRowsSorted(kvs)
-	require.Nil(t, err)
-	require.True(t, prevWriter == w.writer)
-	kvs = []common.KvPair{
-		{
-			Key:   []byte("10"),
-			Val:   []byte("val10"),
-			RowID: 10,
-		},
-		{
-			Key:   []byte("11"),
-			Val:   []byte("val11"),
-			RowID: 11,
-		},
-	}
-	err = w.appendRowsSorted(kvs)
-	require.Nil(t, err)
-	require.False(t, prevWriter == w.writer) // id leap, should flush and create
-	prevWriter = w.writer
-	kvs = []common.KvPair{
-		{
-			Key:   []byte("12"),
-			Val:   []byte("val12"),
-			RowID: 10,
-		},
-		{
-			Key:   []byte("13"),
-			Val:   []byte("val13"),
-			RowID: 11,
-		},
-		{
-			Key:   []byte("15"),
-			Val:   []byte("val15"),
-			RowID: 15,
-		},
-	}
-	err = w.appendRowsSorted(kvs)
-	require.Nil(t, err)
-	require.False(t, prevWriter == w.writer) // id leap, should flush and create
-	prevWriter = w.writer
-	kvs = []common.KvPair{
-		{
-			Key:   []byte("16"),
-			Val:   []byte("val16"),
-			RowID: 16,
-		},
-		{
-			Key:   []byte("17"),
-			Val:   []byte("val17"),
-			RowID: 17,
-		},
-		{
-			Key:   []byte("19"),
-			Val:   []byte("val19"),
-			RowID: 19,
-		},
-		{
-			Key:   []byte("20"),
-			Val:   []byte("val20"),
-			RowID: 20,
-		},
-		{
-			Key:   []byte("22"),
-			Val:   []byte("val22"),
-			RowID: 22,
-		},
-		{
-			Key:   []byte("23"),
-			Val:   []byte("val23"),
-			RowID: 22,
-		},
-	}
-	err = w.appendRowsSorted(kvs)
-	require.Nil(t, err)
-	require.False(t, prevWriter == w.writer) // id leap, should flush and create
-}

br/pkg/lightning/backend/local/local.go

@@ -1504,7 +1504,6 @@ func (local *local) ResolveDuplicateRows(ctx context.Context, tbl table.Table, t
 		logger.Warn("[resolve-dupe] skipping resolution due to selected algorithm. this table will become inconsistent!", zap.Stringer("algorithm", algorithm))
 		return nil
 	case config.DupeResAlgRemove:
-		break
 	default:
 		panic(fmt.Sprintf("[resolve-dupe] unknown resolution algorithm %v", algorithm))
 	}

br/pkg/lightning/config/config.go

@@ -1156,7 +1156,7 @@ func (cfg *Config) CheckAndAdjustSecurity() error {
 			return common.ErrInvalidConfig.GenWithStack("cannot set `tidb.tls` to 'cluster' without a [security] section")
 		}
 	case "false", "skip-verify", "preferred":
-		break
+		return nil
 	default:
 		return common.ErrInvalidConfig.GenWithStack("unsupported `tidb.tls` config %s", cfg.TiDB.TLS)
 	}

br/pkg/lightning/mydump/BUILD.bazel

@@ -31,7 +31,6 @@ go_library(
         "//util/slice",
         "//util/table-filter",
         "@com_github_pingcap_errors//:errors",
-        "@com_github_pingcap_failpoint//:failpoint",
         "@com_github_xitongsys_parquet_go//parquet",
         "@com_github_xitongsys_parquet_go//reader",
         "@com_github_xitongsys_parquet_go//source",

br/pkg/lightning/mydump/loader.go

@@ -39,21 +39,19 @@ type MDDatabaseMeta struct {
 }
 
 func (m *MDDatabaseMeta) GetSchema(ctx context.Context, store storage.ExternalStorage) string {
-	schema, err := ExportStatement(ctx, store, m.SchemaFile, m.charSet)
-	if err != nil {
-		log.FromContext(ctx).Warn("failed to extract table schema",
-			zap.String("Path", m.SchemaFile.FileMeta.Path),
-			log.ShortError(err),
-		)
-		schema = nil
-	}
-	schemaStr := strings.TrimSpace(string(schema))
-	// set default if schema sql is empty
-	if len(schemaStr) == 0 {
-		schemaStr = "CREATE DATABASE IF NOT EXISTS " + common.EscapeIdentifier(m.Name)
+	if m.SchemaFile.FileMeta.Path != "" {
+		schema, err := ExportStatement(ctx, store, m.SchemaFile, m.charSet)
+		if err != nil {
+			log.FromContext(ctx).Warn("failed to extract table schema",
+				zap.String("Path", m.SchemaFile.FileMeta.Path),
+				log.ShortError(err),
+			)
+		} else if schemaStr := strings.TrimSpace(string(schema)); schemaStr != "" {
+			return schemaStr
+		}
 	}
-
-	return schemaStr
+	// set default if schema sql is empty or failed to extract.
+	return "CREATE DATABASE IF NOT EXISTS " + common.EscapeIdentifier(m.Name)
 }
 
 type MDTableMeta struct {

br/pkg/lightning/mydump/loader_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/pingcap/tidb/br/pkg/lightning/config"
+	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	md "github.com/pingcap/tidb/br/pkg/lightning/mydump"
 	"github.com/pingcap/tidb/br/pkg/storage"
 	filter "github.com/pingcap/tidb/util/table-filter"
@@ -181,13 +182,16 @@ func TestTableInfoNotFound(t *testing.T) {
 	loader, err := md.NewMyDumpLoader(ctx, s.cfg)
 	require.NoError(t, err)
 	for _, dbMeta := range loader.GetDatabases() {
-		dbSQL := dbMeta.GetSchema(ctx, store)
+		logger, buffer := log.MakeTestLogger()
+		logCtx := log.NewContext(ctx, logger)
+		dbSQL := dbMeta.GetSchema(logCtx, store)
 		require.Equal(t, "CREATE DATABASE IF NOT EXISTS `db`", dbSQL)
 		for _, tblMeta := range dbMeta.Tables {
-			sql, err := tblMeta.GetSchema(ctx, store)
+			sql, err := tblMeta.GetSchema(logCtx, store)
 			require.Equal(t, "", sql)
 			require.NoError(t, err)
 		}
+		require.NotContains(t, buffer.Stripped(), "failed to extract table schema")
 	}
 }