diff --git a/config/config.go b/config/config.go index c4cb497f66df7..1b7280ab2c84d 100644 --- a/config/config.go +++ b/config/config.go @@ -94,9 +94,10 @@ type Config struct { AlterPrimaryKey bool `toml:"alter-primary-key" json:"alter-primary-key"` // TreatOldVersionUTF8AsUTF8MB4 is use to treat old version table/column UTF8 charset as UTF8MB4. This is for compatibility. // Currently not support dynamic modify, because this need to reload all old version schema. - TreatOldVersionUTF8AsUTF8MB4 bool `toml:"treat-old-version-utf8-as-utf8mb4" json:"treat-old-version-utf8-as-utf8mb4"` - SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` - StmtSummary StmtSummary `toml:"stmt-summary" json:"stmt-summary"` + TreatOldVersionUTF8AsUTF8MB4 bool `toml:"treat-old-version-utf8-as-utf8mb4" json:"treat-old-version-utf8-as-utf8mb4"` + SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` + StmtSummary StmtSummary `toml:"stmt-summary" json:"stmt-summary"` + Experimental Experimental `toml:"experimental" json:"experimental"` } // Log is the log section of config. @@ -335,6 +336,13 @@ type StmtSummary struct { HistorySize int `toml:"history-size" json:"history-size"` } +// Experimental controls the features that are still experimental: their semantics, interfaces are subject to change. +// Using these features in the production environment is not recommended. +type Experimental struct { + // Whether enable the syntax like `auto_random(3)` on the primary key column. + AllowAutoRandom bool `toml:"allow-auto-random" json:"allow-auto-random"` +} + var defaultConf = Config{ Host: "0.0.0.0", AdvertiseAddress: "", @@ -440,6 +448,9 @@ var defaultConf = Config{ RefreshInterval: 1800, HistorySize: 24, }, + Experimental: Experimental{ + AllowAutoRandom: false, + }, } var ( @@ -619,6 +630,10 @@ func (c *Config) Valid() error { if c.StmtSummary.RefreshInterval <= 0 { return fmt.Errorf("refresh-interval in [stmt-summary] should be greater than 0") } + + if c.AlterPrimaryKey && c.Experimental.AllowAutoRandom { + return fmt.Errorf("allow-auto-random is unavailable when alter-primary-key is enabled") + } return nil } diff --git a/config/config.toml.example b/config/config.toml.example index d8cc36f6afd5b..a9d71dcd7e1cb 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -353,3 +353,9 @@ refresh-interval = 1800 # the maximum history size of statement summary. history-size = 24 + +# experimental section controls the features that are still experimental: their semantics, +# interfaces are subject to change, using these features in the production environment is not recommended. +[experimental] +# enable column attribute `auto_random` to be defined on the primary key column. +allow-auto-random = false diff --git a/config/config_test.go b/config/config_test.go index 7d3efe4520452..0732f2d42cb94 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -81,6 +81,8 @@ max-stmt-count=1000 max-sql-length=1024 refresh-interval=100 history-size=100 +[experimental] +allow-auto-random = true `) c.Assert(err, IsNil) @@ -112,6 +114,7 @@ history-size=100 c.Assert(conf.StmtSummary.MaxSQLLength, Equals, uint(1024)) c.Assert(conf.StmtSummary.RefreshInterval, Equals, 100) c.Assert(conf.StmtSummary.HistorySize, Equals, 100) + c.Assert(conf.Experimental.AllowAutoRandom, IsTrue) c.Assert(f.Close(), IsNil) c.Assert(os.Remove(configFile), IsNil) @@ -246,3 +249,16 @@ func (s *testConfigSuite) TestOOMActionValid(c *C) { c.Assert(c1.Valid() == nil, Equals, tt.valid) } } + +func (s *testConfigSuite) TestAllowAutoRandomValid(c *C) { + conf := NewConfig() + checkValid := func(allowAlterPK, allowAutoRand, shouldBeValid bool) { + conf.AlterPrimaryKey = allowAlterPK + conf.Experimental.AllowAutoRandom = allowAutoRand + c.Assert(conf.Valid() == nil, Equals, shouldBeValid) + } + checkValid(true, true, false) + checkValid(true, false, true) + checkValid(false, true, true) + checkValid(false, false, true) +} diff --git a/ddl/column_change_test.go b/ddl/column_change_test.go index 62eb19840cb24..0d49316341c86 100644 --- a/ddl/column_change_test.go +++ b/ddl/column_change_test.go @@ -352,8 +352,8 @@ func getCurrentTable(d *ddl, schemaID, tableID int64) (table.Table, error) { if err != nil { return nil, errors.Trace(err) } - alloc := autoid.NewAllocator(d.store, schemaID, false) - tbl, err := table.TableFromMeta(alloc, tblInfo) + alloc := autoid.NewAllocator(d.store, schemaID, false, autoid.RowIDAllocType) + tbl, err := table.TableFromMeta(autoid.NewAllocators(alloc), tblInfo) if err != nil { return nil, errors.Trace(err) } diff --git a/ddl/ddl.go b/ddl/ddl.go index 65943279665f7..4548e58879bcc 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -234,6 +234,8 @@ var ( ErrFieldNotFoundPart = terror.ClassDDL.New(codeFieldNotFoundPart, mysql.MySQLErrName[mysql.ErrFieldNotFoundPart]) // ErrWrongTypeColumnValue returns 'Partition column values of incorrect type' ErrWrongTypeColumnValue = terror.ClassDDL.New(codeWrongTypeColumnValue, mysql.MySQLErrName[mysql.ErrWrongTypeColumnValue]) + // ErrInvalidAutoRandom returns when auto_random is used incorrectly. + ErrInvalidAutoRandom = terror.ClassDDL.New(codeInvalidAutoRandom, mysql.MySQLErrName[mysql.ErrInvalidAutoRandom]) ) // DDL is responsible for updating schema in data store and maintaining in-memory InfoSchema cache. @@ -757,6 +759,7 @@ const ( codeSystemVersioningWrongPartitions = terror.ErrCode(mysql.ErrSystemVersioningWrongPartitions) codeWrongPartitionTypeExpectedSystemTime = terror.ErrCode(mysql.ErrWrongPartitionTypeExpectedSystemTime) codeWrongTypeColumnValue = terror.ErrCode(mysql.ErrWrongTypeColumnValue) + codeInvalidAutoRandom = terror.ErrCode(mysql.ErrInvalidAutoRandom) ) func init() { @@ -831,6 +834,7 @@ func init() { codeWrongTypeColumnValue: mysql.ErrWrongTypeColumnValue, mysql.ErrFkColumnCannotDrop: mysql.ErrFkColumnCannotDrop, mysql.ErrFKIncompatibleColumns: mysql.ErrFKIncompatibleColumns, + mysql.ErrInvalidAutoRandom: mysql.ErrInvalidAutoRandom, } terror.ErrClassToMySQLCodes[terror.ClassDDL] = ddlMySQLErrCodes } diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index aadcf1ed4fa6e..40ca14e0e1bc1 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -876,9 +876,9 @@ func checkDuplicateColumn(cols []interface{}) error { return nil } -func checkIsAutoIncrementColumn(colDefs *ast.ColumnDef) bool { - for _, option := range colDefs.Options { - if option.Tp == ast.ColumnOptionAutoIncrement { +func containsColumnOption(colDef *ast.ColumnDef, opTp ast.ColumnOptionType) bool { + for _, option := range colDef.Options { + if option.Tp == opTp { return true } } @@ -897,7 +897,7 @@ func checkGeneratedColumn(colDefs []*ast.ColumnDef) error { } } } - if checkIsAutoIncrementColumn(colDef) { + if containsColumnOption(colDef, ast.ColumnOptionAutoIncrement) { exists, autoIncrementColumn = true, colDef.Name.Name.L } generated, depCols := findDependedColumnNames(colDef) @@ -1076,6 +1076,58 @@ func checkConstraintNames(constraints []*ast.Constraint) error { return nil } +func setTableAutoRandomBits(tbInfo *model.TableInfo, colDefs []*ast.ColumnDef) error { + allowAutoRandom := config.GetGlobalConfig().Experimental.AllowAutoRandom + pkColName := tbInfo.GetPkName() + for _, col := range colDefs { + if containsColumnOption(col, ast.ColumnOptionAutoRandom) { + if !allowAutoRandom { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomExperimentalDisabledErrMsg) + } + if !tbInfo.PKIsHandle || col.Name.Name.L != pkColName.L { + return ErrInvalidAutoRandom.GenWithStackByArgs(fmt.Sprintf(autoid.AutoRandomPKisNotHandleErrMsg, col.Name.Name.O)) + } + if containsColumnOption(col, ast.ColumnOptionAutoIncrement) { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg) + } + if containsColumnOption(col, ast.ColumnOptionDefaultValue) { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg) + } + + autoRandBits, err := extractAutoRandomBitsFromColDef(col) + if err != nil { + return errors.Trace(err) + } + maxFieldTypeBitsLength := uint64(mysql.DefaultLengthOfMysqlTypes[col.Tp.Tp] * 8) + if autoRandBits == 0 { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNonPositive) + } else if autoRandBits >= maxFieldTypeBitsLength { + return ErrInvalidAutoRandom.GenWithStackByArgs(fmt.Sprintf(autoid.AutoRandomOverflowErrMsg, autoRandBits, maxFieldTypeBitsLength)) + } + tbInfo.AutoRandomBits = autoRandBits + } + } + return nil +} + +func extractAutoRandomBitsFromColDef(colDef *ast.ColumnDef) (uint64, error) { + for _, op := range colDef.Options { + if op.Tp == ast.ColumnOptionAutoRandom { + return convertAutoRandomBitsToUnsigned(op.AutoRandomBitLength) + } + } + return 0, nil +} + +func convertAutoRandomBitsToUnsigned(autoRandomBits int) (uint64, error) { + if autoRandomBits == types.UnspecifiedLength { + return autoid.DefaultAutoRandomBits, nil + } else if autoRandomBits < 0 { + return 0, ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNonPositive) + } + return uint64(autoRandomBits), nil +} + func buildTableInfo(ctx sessionctx.Context, d *ddl, tableName model.CIStr, cols []*table.Column, constraints []*ast.Constraint) (tbInfo *model.TableInfo, err error) { tbInfo = &model.TableInfo{ Name: tableName, @@ -1152,7 +1204,7 @@ func buildTableInfo(ctx sessionctx.Context, d *ddl, tableName model.CIStr, cols if err != nil { return nil, errors.Trace(err) } - //check if the index is primary or uniqiue. + // check if the index is primary or unique. switch constr.Tp { case ast.ConstraintPrimaryKey: idxInfo.Primary = true @@ -1323,6 +1375,10 @@ func buildTableInfoWithCheck(ctx sessionctx.Context, d *ddl, s *ast.CreateTableS tbInfo.Collate = tableCollate tbInfo.Charset = tableCharset + if err = setTableAutoRandomBits(tbInfo, colDefs); err != nil { + return nil, errors.Trace(err) + } + pi, err := buildTablePartitionInfo(ctx, d, s) if err != nil { return nil, errors.Trace(err) @@ -1719,9 +1775,9 @@ func checkCharsetAndCollation(cs string, co string) error { // handleAutoIncID handles auto_increment option in DDL. It creates a ID counter for the table and initiates the counter to a proper value. // For example if the option sets auto_increment to 10. The counter will be set to 9. So the next allocated ID will be 10. func (d *ddl) handleAutoIncID(tbInfo *model.TableInfo, schemaID int64) error { - alloc := autoid.NewAllocator(d.store, tbInfo.GetDBID(schemaID), tbInfo.IsAutoIncColUnsigned()) + allocs := autoid.NewAllocatorsFromTblInfo(d.store, schemaID, tbInfo) tbInfo.State = model.StatePublic - tb, err := table.TableFromMeta(alloc, tbInfo) + tb, err := table.TableFromMeta(allocs, tbInfo) if err != nil { return errors.Trace(err) } @@ -1978,7 +2034,7 @@ func (d *ddl) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int6 if err != nil { return errors.Trace(err) } - autoIncID, err := t.Allocator(ctx).NextGlobalAutoID(t.Meta().ID) + autoIncID, err := t.Allocator(ctx, autoid.RowIDAllocType).NextGlobalAutoID(t.Meta().ID) if err != nil { return errors.Trace(err) } @@ -2524,6 +2580,8 @@ func processColumnOptions(ctx sessionctx.Context, col *table.Column, options []* return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs("with references")) case ast.ColumnOptionFulltext: return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs("with full text")) + // Ignore ColumnOptionAutoRandom. It will be handled later. + case ast.ColumnOptionAutoRandom: default: return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs(fmt.Sprintf("unknown column option type: %d", opt.Tp))) } @@ -2668,6 +2726,10 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or return nil, errors.Trace(err) } + if err = checkAutoRandom(t.Meta(), col, specNewColumn); err != nil { + return nil, errors.Trace(err) + } + job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, @@ -2715,6 +2777,36 @@ func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol return nil } +func checkAutoRandom(tableInfo *model.TableInfo, originCol *table.Column, specNewColumn *ast.ColumnDef) error { + if !config.GetGlobalConfig().Experimental.AllowAutoRandom && containsColumnOption(specNewColumn, ast.ColumnOptionAutoRandom) { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomExperimentalDisabledErrMsg) + } + // Disallow add/drop/modify actions on auto_random. + newAutoRandomBit, err := extractAutoRandomBitsFromColDef(specNewColumn) + if err != nil { + return errors.Trace(err) + } + if tableInfo.AutoRandomBits != newAutoRandomBit { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomAlterErrMsg) + } + + if tableInfo.AutoRandomBits != 0 { + // Disallow changing the column field type. + if originCol.Tp != specNewColumn.Tp.Tp { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomModifyColTypeErrMsg) + } + // Disallow changing auto_increment on auto_random column. + if containsColumnOption(specNewColumn, ast.ColumnOptionAutoIncrement) != mysql.HasAutoIncrementFlag(originCol.Flag) { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg) + } + // Disallow specifying a default value on auto_random column. + if containsColumnOption(specNewColumn, ast.ColumnOptionDefaultValue) { + return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg) + } + } + return nil +} + // ChangeColumn renames an existing column and modifies the column's definition, // currently we only support limited kind of changes // that do not need to change or check data on the table. diff --git a/ddl/serial_test.go b/ddl/serial_test.go index 6b6d73d2d86fd..2cf7c3b7c5f9b 100644 --- a/ddl/serial_test.go +++ b/ddl/serial_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/mockstore" @@ -40,6 +41,7 @@ import ( "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/testkit" + "github.com/pingcap/tidb/util/testutil" ) var _ = SerialSuites(&testSerialSuite{}) @@ -743,3 +745,131 @@ func (s *testSerialSuite) TestCanceledJobTakeTime(c *C) { sub := time.Since(startTime) c.Assert(sub, Less, ddl.WaitTimeWhenErrorOccured) } + +func (s *testSerialSuite) TestAutoRandom(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists auto_random_db") + defer tk.MustExec("drop database if exists auto_random_db") + tk.MustExec("use auto_random_db") + tk.MustExec("drop table if exists t") + + assertInvalidAutoRandomErr := func(sql string, errMsg string, args ...interface{}) { + _, err := tk.Exec(sql) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, ddl.ErrInvalidAutoRandom.GenWithStackByArgs(fmt.Sprintf(errMsg, args...)).Error()) + } + + assertPKIsNotHandle := func(sql, errCol string) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomPKisNotHandleErrMsg, errCol) + } + assertExperimentDisabled := func(sql string) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomExperimentalDisabledErrMsg) + } + assertAlterValue := func(sql string) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomAlterErrMsg) + } + assertWithAutoInc := func(sql string) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomIncompatibleWithAutoIncErrMsg) + } + assertOverflow := func(sql string, autoRandBits, maxFieldLength uint64) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomOverflowErrMsg, autoRandBits, maxFieldLength) + } + assertModifyColType := func(sql string) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomModifyColTypeErrMsg) + } + assertDefault := func(sql string) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomIncompatibleWithDefaultValueErrMsg) + } + assertNonPositive := func(sql string) { + assertInvalidAutoRandomErr(sql, autoid.AutoRandomNonPositive) + } + mustExecAndDrop := func(sql string, fns ...func()) { + tk.MustExec(sql) + for _, f := range fns { + f() + } + tk.MustExec("drop table t") + } + + testutil.ConfigTestUtils.SetupAutoRandomTestConfig() + defer testutil.ConfigTestUtils.RestoreAutoRandomTestConfig() + // PKIsHandle, but auto_random is defined on non-primary key. + assertPKIsNotHandle("create table t (a bigint auto_random (3) primary key, b int auto_random (3))", "b") + assertPKIsNotHandle("create table t (a bigint auto_random (3), b int auto_random(3), primary key(a))", "b") + assertPKIsNotHandle("create table t (a bigint auto_random (3), b int auto_random(3) primary key)", "a") + + // PKIsNotHandle: no primary key. + assertPKIsNotHandle("create table t (a int auto_random(3), b int)", "a") + assertPKIsNotHandle("create table t (a bigint auto_random(3), b int)", "a") + // PKIsNotHandle: primary key is not integer column. + assertPKIsNotHandle("create table t (a char primary key auto_random(3), b int)", "a") + assertPKIsNotHandle("create table t (a varchar(255) primary key auto_random(3), b int)", "a") + assertPKIsNotHandle("create table t (a timestamp primary key auto_random(3), b int)", "a") + // PKIsNotHandle: primary key is not a single column. + assertPKIsNotHandle("create table t (a bigint auto_random(3), b int, primary key (a, b))", "a") + assertPKIsNotHandle("create table t (a int auto_random(3), b int, c char, primary key (a, c))", "a") + + // Can not set auto_random along with auto_increment. + assertWithAutoInc("create table t (a bigint auto_random(3) primary key auto_increment)") + assertWithAutoInc("create table t (a bigint primary key auto_increment auto_random(3))") + assertWithAutoInc("create table t (a bigint auto_increment primary key auto_random(3))") + assertWithAutoInc("create table t (a bigint auto_random(3) auto_increment, primary key (a))") + + // Overflow data type max length. + assertOverflow("create table t (a bigint auto_random(65) primary key)", 65, 64) + assertOverflow("create table t (a int auto_random(33) primary key)", 33, 32) + assertOverflow("create table t (a mediumint auto_random(25) primary key)", 25, 24) + assertOverflow("create table t (a smallint auto_random(17) primary key)", 17, 16) + assertOverflow("create table t (a tinyint auto_random(9) primary key)", 9, 8) + + assertNonPositive("create table t (a bigint auto_random(0) primary key)") + + // Can not set auto_random along with default. + assertDefault("create table t (a int auto_random primary key default 3)") + assertDefault("create table t (a bigint auto_random(2) primary key default 5)") + mustExecAndDrop("create table t (a int auto_random primary key)", func() { + assertDefault("alter table t modify column a int auto_random default 3") + }) + + // Basic usage. + mustExecAndDrop("create table t (a bigint auto_random(4) primary key, b varchar(255))") + mustExecAndDrop("create table t (a bigint primary key auto_random(4), b varchar(255))") + mustExecAndDrop("create table t (a bigint auto_random(4), b varchar(255), primary key (a))") + + // Different primary key field types. + mustExecAndDrop("create table t (a bigint auto_random(4) primary key)") + mustExecAndDrop("create table t (a int auto_random(4) primary key)") + mustExecAndDrop("create table t (a mediumint auto_random(4) primary key)") + mustExecAndDrop("create table t (a smallint auto_random(4) primary key)") + mustExecAndDrop("create table t (a tinyint auto_random(4) primary key)") + + // Auto_random can occur multiple times like other column attributes. + mustExecAndDrop("create table t (a bigint auto_random(3) auto_random(2) primary key)") + mustExecAndDrop("create table t (a int, b bigint auto_random(3) primary key auto_random(2))") + mustExecAndDrop("create table t (a int auto_random(1) auto_random(2) auto_random(3), primary key (a))") + + // Add/drop the auto_random attribute is not allowed. + mustExecAndDrop("create table t (a bigint auto_random(3) primary key)", func() { + assertAlterValue("alter table t modify column a bigint") + assertAlterValue("alter table t change column a b bigint") + }) + mustExecAndDrop("create table t (a int, b char, c int auto_random(3), primary key(c))", func() { + assertAlterValue("alter table t modify column c bigint") + assertAlterValue("alter table t change column c d bigint") + }) + mustExecAndDrop("create table t (a bigint primary key)", func() { + assertAlterValue("alter table t modify column a bigint auto_random(3)") + assertAlterValue("alter table t change column a b bigint auto_random(3)") + }) + + // Modifying the field type of a auto_random column is not allowed. + mustExecAndDrop("create table t (a tinyint primary key auto_random(3))", func() { + assertModifyColType("alter table t modify column a int auto_random(3)") + assertModifyColType("alter table t modify column a mediumint auto_random(3)") + assertModifyColType("alter table t modify column a smallint auto_random(3)") + }) + + // Disallow using it when allow-auto-random is not enabled. + config.GetGlobalConfig().Experimental.AllowAutoRandom = false + assertExperimentDisabled("create table auto_random_table (a int primary key auto_random(3))") +} diff --git a/ddl/table.go b/ddl/table.go index 447cd9fa41316..c0b9d0bd9d4c5 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -339,8 +339,8 @@ func checkSafePoint(w *worker, snapshotTS uint64) error { } func getTable(store kv.Storage, schemaID int64, tblInfo *model.TableInfo) (table.Table, error) { - alloc := autoid.NewAllocator(store, tblInfo.GetDBID(schemaID), tblInfo.IsAutoIncColUnsigned()) - tbl, err := table.TableFromMeta(alloc, tblInfo) + allocs := autoid.NewAllocatorsFromTblInfo(store, schemaID, tblInfo) + tbl, err := table.TableFromMeta(allocs, tblInfo) return tbl, errors.Trace(err) } @@ -520,11 +520,11 @@ func verifyNoOverflowShardBits(s *sessionPool, tbl table.Table, shardRowIDBits u defer s.put(ctx) // Check next global max auto ID first. - autoIncID, err := tbl.Allocator(ctx).NextGlobalAutoID(tbl.Meta().ID) + autoIncID, err := tbl.Allocator(ctx, autoid.RowIDAllocType).NextGlobalAutoID(tbl.Meta().ID) if err != nil { return errors.Trace(err) } - if tables.OverflowShardBits(autoIncID, shardRowIDBits) { + if tables.OverflowShardBits(autoIncID, shardRowIDBits, autoid.RowIDBitLength) { return autoid.ErrAutoincReadFailed.GenWithStack("shard_row_id_bits %d will cause next global auto ID %v overflow", shardRowIDBits, autoIncID) } return nil diff --git a/ddl/table_test.go b/ddl/table_test.go index ca189139b7d9d..69528c395a1f0 100644 --- a/ddl/table_test.go +++ b/ddl/table_test.go @@ -256,8 +256,8 @@ func testGetTableWithError(d *ddl, schemaID, tableID int64) (table.Table, error) if tblInfo == nil { return nil, errors.New("table not found") } - alloc := autoid.NewAllocator(d.store, schemaID, false) - tbl, err := table.TableFromMeta(alloc, tblInfo) + alloc := autoid.NewAllocator(d.store, schemaID, false, autoid.RowIDAllocType) + tbl, err := table.TableFromMeta(autoid.NewAllocators(alloc), tblInfo) if err != nil { return nil, errors.Trace(err) } diff --git a/executor/ddl_test.go b/executor/ddl_test.go index c0d6b20711312..1785f827db49c 100644 --- a/executor/ddl_test.go +++ b/executor/ddl_test.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "math" + "sort" "strconv" "strings" "time" @@ -633,8 +634,9 @@ func (s *testSuite3) TestShardRowIDBits(c *C) { tk.MustExec("use test") tk.MustExec("create table t (a int) shard_row_id_bits = 15") for i := 0; i < 100; i++ { - tk.MustExec(fmt.Sprintf("insert t values (%d)", i)) + tk.MustExec("insert into t values (?)", i) } + dom := domain.GetDomain(tk.Se) tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) @@ -708,7 +710,7 @@ func (s *testSuite3) TestShardRowIDBits(c *C) { // Test shard_row_id_bits with auto_increment column tk.MustExec("create table auto (a int, b int auto_increment unique) shard_row_id_bits = 15") for i := 0; i < 100; i++ { - tk.MustExec(fmt.Sprintf("insert auto(a) values (%d)", i)) + tk.MustExec("insert into auto(a) values (?)", i) } tbl, err = dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("auto")) assertCountAndShard(tbl, 100) @@ -740,6 +742,94 @@ func (s *testSuite3) TestShardRowIDBits(c *C) { c.Assert(autoid.ErrAutoincReadFailed.Equal(err), IsTrue, Commentf("err:%v", err)) } +type testAutoRandomSuite struct { + testSuite +} + +func (s *testAutoRandomSuite) TestAutoRandomBitsData(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("create database if not exists test_auto_random_bits") + defer tk.MustExec("drop database if exists test_auto_random_bits") + tk.MustExec("use test_auto_random_bits") + tk.MustExec("drop table if exists t") + + testutil.ConfigTestUtils.SetupAutoRandomTestConfig() + defer testutil.ConfigTestUtils.RestoreAutoRandomTestConfig() + + tk.MustExec("create table t (a bigint primary key auto_random(15), b int)") + for i := 0; i < 100; i++ { + tk.MustExec("insert into t(b) values (?)", i) + } + dom := domain.GetDomain(tk.Se) + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test_auto_random_bits"), model.NewCIStr("t")) + c.Assert(err, IsNil) + c.Assert(tk.Se.NewTxn(context.Background()), IsNil) + var allHandles []int64 + // Iterate all the record. The order is not guaranteed. + err = tbl.IterRecords(tk.Se, tbl.FirstKey(), nil, func(h int64, _ []types.Datum, _ []*table.Column) (more bool, err error) { + allHandles = append(allHandles, h) + return true, nil + }) + c.Assert(err, IsNil) + tk.MustExec("drop table t") + + // Test auto random id number. + c.Assert(len(allHandles), Equals, 100) + // Test the first 15 bits of each handle is greater than 0. + for _, h := range allHandles { + c.Assert(h>>(64-15), Greater, int64(0)) + } + // Test auto random id is monotonic increasing and continuous. + orderedHandles := make([]int64, len(allHandles)) + for i, h := range allHandles { + orderedHandles[i] = h << 16 >> 16 + } + sort.Slice(orderedHandles, func(i, j int) bool { return orderedHandles[i] < orderedHandles[j] }) + size := int64(len(allHandles)) + for i := int64(1); i <= size; i++ { + c.Assert(i, Equals, orderedHandles[i-1]) + } + + // Test explicit insert. + tk.MustExec("create table t (a tinyint primary key auto_random(2), b int)") + for i := 0; i < 100; i++ { + tk.MustExec("insert into t values (?, ?)", i, i) + } + _, err = tk.Exec("insert into t (b) values (0)") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, autoid.ErrAutoRandReadFailed.GenWithStackByArgs().Error()) + tk.MustExec("drop table t") + + // Test overflow. + tk.MustExec("create table t (a tinyint primary key auto_random(2), b int)") + fieldLength := uint64(mysql.DefaultLengthOfMysqlTypes[mysql.TypeTiny] * 8) + signBit := uint64(1) + for i := 0; i < (1<<(fieldLength-2-signBit))-1; i++ { + tk.MustExec(fmt.Sprintf("insert into t (b) values (%d)", i)) + } + _, err = tk.Exec("insert into t (b) values (0)") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, autoid.ErrAutoRandReadFailed.GenWithStackByArgs().Error()) + tk.MustExec("drop table t") + + // Test rebase. + tk.MustExec("create table t (a tinyint primary key auto_random(2), b int)") + tk.MustExec("insert into t values (31, 2)") + _, err = tk.Exec("insert into t (b) values (0)") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, autoid.ErrAutoRandReadFailed.GenWithStackByArgs().Error()) + tk.MustExec("drop table t") + + tk.MustExec("create table t (a tinyint primary key auto_random(2), b int)") + tk.MustExec("insert into t values (0, 2)") + tk.MustExec("update t set a = 31 where a = 0") + _, err = tk.Exec("insert into t (b) values (0)") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, autoid.ErrAutoRandReadFailed.GenWithStackByArgs().Error()) + tk.MustExec("drop table t") +} + func (s *testSuite3) TestMaxHandleAddIndex(c *C) { tk := testkit.NewTestKit(c, s.store) diff --git a/executor/executor.go b/executor/executor.go index 74f3077687ce7..e36315c0558b4 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/meta/autoid" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" @@ -249,7 +250,7 @@ func (e *ShowNextRowIDExec) Next(ctx context.Context, req *chunk.Chunk) error { break } } - nextGlobalID, err := tbl.Allocator(e.ctx).NextGlobalAutoID(tbl.Meta().ID) + nextGlobalID, err := tbl.Allocator(e.ctx, autoid.RowIDAllocType).NextGlobalAutoID(tbl.Meta().ID) if err != nil { return err } diff --git a/executor/executor_test.go b/executor/executor_test.go index a1e54d17c99f1..49f26af521c9c 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -97,6 +97,7 @@ var _ = Suite(&testOOMSuite{}) var _ = Suite(&testPointGetSuite{}) var _ = Suite(&testFlushSuite{}) var _ = SerialSuites(&testShowStatsSuite{}) +var _ = SerialSuites(&testAutoRandomSuite{testSuite{}}) type testSuite struct { cluster *mocktikv.Cluster @@ -3024,8 +3025,8 @@ func (s *testSuite) TestCheckIndex(c *C) { c.Assert(err, IsNil) tbInfo := tbl.Meta() - alloc := autoid.NewAllocator(s.store, dbInfo.ID, false) - tb, err := tables.TableFromMeta(alloc, tbInfo) + alloc := autoid.NewAllocator(s.store, dbInfo.ID, false, autoid.RowIDAllocType) + tb, err := tables.TableFromMeta(autoid.NewAllocators(alloc), tbInfo) c.Assert(err, IsNil) _, err = se.Execute(context.Background(), "admin check index t c") diff --git a/executor/insert_common.go b/executor/insert_common.go index a043e8316b9f7..9840e83225b8a 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -24,7 +24,9 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" @@ -421,6 +423,14 @@ func (e *InsertValues) fillColValue(ctx context.Context, datum types.Datum, idx } return d, nil } + tblInfo := e.Table.Meta() + if tblInfo.PKIsHandle && tblInfo.ContainsAutoRandomBits() && column.ID == tblInfo.GetPkColInfo().ID { + d, err := e.adjustAutoRandomDatum(ctx, datum, hasValue, column) + if err != nil { + return types.Datum{}, err + } + return d, nil + } if !hasValue { d, err := e.getColDefaultValue(idx, column) if e.filterErr(err) != nil { @@ -706,6 +716,66 @@ func getAutoRecordID(d types.Datum, target *types.FieldType, isInsert bool) (int return recordID, nil } +func (e *InsertValues) adjustAutoRandomDatum(ctx context.Context, d types.Datum, hasValue bool, c *table.Column) (types.Datum, error) { + if !hasValue || d.IsNull() { + _, err := e.ctx.Txn(true) + if err != nil { + return types.Datum{}, errors.Trace(err) + } + autoRandomID, err := e.allocAutoRandomID(&c.FieldType) + if err != nil { + return types.Datum{}, err + } + d.SetAutoID(autoRandomID, c.Flag) + } else { + recordID, err := getAutoRecordID(d, &c.FieldType, true) + if err != nil { + return types.Datum{}, err + } + err = e.rebaseAutoRandomID(recordID, &c.FieldType) + if err != nil { + return types.Datum{}, err + } + d.SetAutoID(recordID, c.Flag) + } + + casted, err := table.CastValue(e.ctx, d, c.ToInfo()) + if err != nil { + return types.Datum{}, err + } + return casted, nil +} + +// allocAutoRandomID allocates a random id for primary key column. It assumes tableInfo.AutoRandomBits > 0. +func (e *InsertValues) allocAutoRandomID(fieldType *types.FieldType) (int64, error) { + alloc := e.Table.Allocator(e.ctx, autoid.AutoRandomType) + tableInfo := e.Table.Meta() + _, autoRandomID, err := alloc.Alloc(tableInfo.ID, 1) + if err != nil { + return 0, err + } + + typeBitsLength := uint64(mysql.DefaultLengthOfMysqlTypes[fieldType.Tp] * 8) + if tables.OverflowShardBits(autoRandomID, tableInfo.AutoRandomBits, typeBitsLength) { + return 0, autoid.ErrAutoRandReadFailed + } + shard := tables.CalcShard(tableInfo.AutoRandomBits, e.ctx.GetSessionVars().TxnCtx.StartTS, typeBitsLength) + autoRandomID |= shard + return autoRandomID, nil +} + +func (e *InsertValues) rebaseAutoRandomID(recordID int64, fieldType *types.FieldType) error { + alloc := e.Table.Allocator(e.ctx, autoid.AutoRandomType) + tableInfo := e.Table.Meta() + + typeBitsLength := uint64(mysql.DefaultLengthOfMysqlTypes[fieldType.Tp] * 8) + signBit := uint64(1) + mask := (1 << (typeBitsLength - tableInfo.AutoRandomBits - signBit)) - 1 + autoRandomID := int64(mask) & recordID + + return alloc.Rebase(tableInfo.ID, autoRandomID, true) +} + func (e *InsertValues) handleWarning(err error) { sc := e.ctx.GetSessionVars().StmtCtx sc.AppendWarning(err) diff --git a/executor/show.go b/executor/show.go index 8aed8bbce8f33..75cecdd4258fc 100644 --- a/executor/show.go +++ b/executor/show.go @@ -25,6 +25,7 @@ import ( "github.com/cznic/mathutil" "github.com/pingcap/errors" + "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/charset" @@ -733,8 +734,11 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T buf.WriteString(table.OptionalFsp(&col.FieldType)) } } + if tableInfo.PKIsHandle && tableInfo.ContainsAutoRandomBits() && tableInfo.GetPkName().L == col.Name.L { + buf.WriteString(fmt.Sprintf(" /*T!%s AUTO_RANDOM(%d) */", parser.CommentCodeAutoRandom, tableInfo.AutoRandomBits)) + } if len(col.Comment) > 0 { - fmt.Fprintf(buf, " COMMENT '%s'", format.OutputFormat(col.Comment)) + buf.WriteString(fmt.Sprintf(" COMMENT '%s'", format.OutputFormat(col.Comment))) } if i != len(tableInfo.Cols())-1 { buf.WriteString(",\n") @@ -835,7 +839,7 @@ func (e *ShowExec) fetchShowCreateTable() error { } tableInfo := tb.Meta() - allocator := tb.Allocator(e.ctx) + allocator := tb.Allocator(e.ctx, autoid.RowIDAllocType) var buf bytes.Buffer // TODO: let the result more like MySQL. if err = ConstructResultOfShowCreateTable(e.ctx, tb.Meta(), allocator, &buf); err != nil { diff --git a/executor/show_test.go b/executor/show_test.go index ef08c6d7c40f7..f39e9b67a08b8 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -594,6 +594,7 @@ func (s *testSuite2) TestShowCreateTable(c *C) { " PARTITION p11 VALUES LESS THAN (12),\n"+ " PARTITION p12 VALUES LESS THAN (MAXVALUE)\n"+ ")")) + //for issue #11831 tk.MustExec("create table ttt4(a varchar(123) default null collate utf8mb4_unicode_ci)engine=innodb default charset=utf8mb4 collate=utf8mb4_unicode_ci;") tk.MustQuery("show create table `ttt4`").Check(testutil.RowsWithSep("|", @@ -639,3 +640,43 @@ func (s *testSuite2) TestShowEscape(c *C) { tk.MustExec("rename table \"t`abl\"\"e\" to t") tk.MustExec("set sql_mode=@old_sql_mode") } + +func (s *testAutoRandomSuite) TestShowCreateTableAutoRandom(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + testutil.ConfigTestUtils.SetupAutoRandomTestConfig() + defer testutil.ConfigTestUtils.RestoreAutoRandomTestConfig() + + // Basic show create table. + tk.MustExec("create table auto_random_tbl1 (a bigint primary key auto_random(3), b varchar(255))") + tk.MustQuery("show create table `auto_random_tbl1`").Check(testutil.RowsWithSep("|", + ""+ + "auto_random_tbl1 CREATE TABLE `auto_random_tbl1` (\n"+ + " `a` bigint(20) NOT NULL /*T!40000 AUTO_RANDOM(3) */,\n"+ + " `b` varchar(255) DEFAULT NULL,\n"+ + " PRIMARY KEY (`a`)\n"+ + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin", + )) + + // Implicit auto_random value should be shown explicitly. + tk.MustExec("create table auto_random_tbl2 (a bigint auto_random primary key, b char)") + tk.MustQuery("show create table auto_random_tbl2").Check(testutil.RowsWithSep("|", + ""+ + "auto_random_tbl2 CREATE TABLE `auto_random_tbl2` (\n"+ + " `a` bigint(20) NOT NULL /*T!40000 AUTO_RANDOM(5) */,\n"+ + " `b` char(1) DEFAULT NULL,\n"+ + " PRIMARY KEY (`a`)\n"+ + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin", + )) + + // Special version comment can be shown in TiDB with new version. + tk.MustExec("create table auto_random_tbl3 (a bigint /*T!40000 auto_random */ primary key)") + tk.MustQuery("show create table auto_random_tbl3").Check(testutil.RowsWithSep("|", + ""+ + "auto_random_tbl3 CREATE TABLE `auto_random_tbl3` (\n"+ + " `a` bigint(20) NOT NULL /*T!40000 AUTO_RANDOM(5) */,\n"+ + " PRIMARY KEY (`a`)\n"+ + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin", + )) +} diff --git a/executor/write.go b/executor/write.go index 09506aed09587..8be20b9449172 100644 --- a/executor/write.go +++ b/executor/write.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -100,6 +101,10 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu if col.IsPKHandleColumn(t.Meta()) { handleChanged = true newHandle = newData[i].GetInt64() + // Rebase auto random id if the field is changed. + if err := rebaseAutoRandomValue(ctx, t, &newData[i], col); err != nil { + return false, false, 0, err + } } } else { if mysql.HasOnUpdateNowFlag(col.Flag) && modified[i] { @@ -177,6 +182,20 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu return true, handleChanged, newHandle, nil } +func rebaseAutoRandomValue(sctx sessionctx.Context, t table.Table, newData *types.Datum, col *table.Column) error { + tableInfo := t.Meta() + if !tableInfo.ContainsAutoRandomBits() { + return nil + } + recordID, err := getAutoRecordID(*newData, &col.FieldType, false) + if err != nil { + return err + } + shardBits := tableInfo.AutoRandomBits + 1 // sign bit is reserved. + recordID = recordID << shardBits >> shardBits + return t.Allocator(sctx, autoid.AutoRandomType).Rebase(tableInfo.ID, recordID, true) +} + // resetErrDataTooLong reset ErrDataTooLong error msg. // types.ErrDataTooLong is produced in types.ProduceStrWithSpecifiedTp, there is no column info in there, // so we reset the error msg here, and wrap old err with errors.Wrap. diff --git a/go.mod b/go.mod index f5194973a316c..1f7071e8c23fa 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 - github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 + github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f github.com/dustin/go-humanize v1.0.0 // indirect github.com/go-ole/go-ole v1.2.1 // indirect @@ -36,7 +36,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20191113105027-4f292e1801d8 github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd - github.com/pingcap/parser v0.0.0-20200107110252-aba886048346 + github.com/pingcap/parser v0.0.0-20200120071915-21406be5517f github.com/pingcap/pd v1.1.0-beta.0.20191231051840-d1d917558629 github.com/pingcap/tidb-tools v3.0.6-0.20191119150227-ff0a3c6e5763+incompatible github.com/pingcap/tipb v0.0.0-20191126033718-169898888b24 diff --git a/go.sum b/go.sum index b84d04d59daac..dd69f7cb13977 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mo github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -21,17 +22,19 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGda1NeOBQwqlv7nUXpm+rIVHGxZZ4= +github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -180,8 +183,8 @@ github.com/pingcap/kvproto v0.0.0-20191113105027-4f292e1801d8 h1:P9jGgwVkLHlbEGt github.com/pingcap/kvproto v0.0.0-20191113105027-4f292e1801d8/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20200107110252-aba886048346 h1:9+R3klIA/8uOMzVKsHgkoQv3Zng2O8Mk7yuSVYxijkg= -github.com/pingcap/parser v0.0.0-20200107110252-aba886048346/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20200120071915-21406be5517f h1:t9EmvIlBNAqxbS1xvJ2eF3xAxEFgDNUEd2R+K4DZJDQ= +github.com/pingcap/parser v0.0.0-20200120071915-21406be5517f/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v1.1.0-beta.0.20191231051840-d1d917558629 h1:15Rc+aOCosYjquaOI/KoRMNJTz+UEpWFLzrG0XIETRE= github.com/pingcap/pd v1.1.0-beta.0.20191231051840-d1d917558629/go.mod h1:anCie9mDfRyYkso7qjHAgIUFZaXp1S9XI/p2RmmvZq8= github.com/pingcap/tidb-tools v3.0.6-0.20191119150227-ff0a3c6e5763+incompatible h1:I8HirWsu1MZp6t9G/g8yKCEjJJxtHooKakEgccvdJ4M= diff --git a/infoschema/builder.go b/infoschema/builder.go index 72d0d9ce29e72..cf76c64e8be67 100644 --- a/infoschema/builder.go +++ b/infoschema/builder.go @@ -75,10 +75,10 @@ func (b *Builder) ApplyDiff(m *meta.Meta, diff *model.SchemaDiff) ([]int64, erro b.copySortedTables(oldTableID, newTableID) // We try to reuse the old allocator, so the cached auto ID can be reused. - var alloc autoid.Allocator + var allocs autoid.Allocators if tableIDIsValid(oldTableID) { if oldTableID == newTableID && diff.Type != model.ActionRenameTable && diff.Type != model.ActionRebaseAutoID { - alloc, _ = b.is.AllocByID(oldTableID) + allocs, _ = b.is.AllocByID(oldTableID) } if diff.Type == model.ActionRenameTable && diff.OldSchemaID != diff.SchemaID { oldRoDBInfo, ok := b.is.SchemaByID(diff.OldSchemaID) @@ -95,7 +95,7 @@ func (b *Builder) ApplyDiff(m *meta.Meta, diff *model.SchemaDiff) ([]int64, erro } if tableIDIsValid(newTableID) { // All types except DropTableOrView. - err := b.applyCreateTable(m, dbInfo, newTableID, alloc) + err := b.applyCreateTable(m, dbInfo, newTableID, allocs) if err != nil { return nil, errors.Trace(err) } @@ -179,7 +179,7 @@ func (b *Builder) copySortedTablesBucket(bucketIdx int) { b.is.sortedTablesBuckets[bucketIdx] = newSortedTables } -func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID int64, alloc autoid.Allocator) error { +func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID int64, allocs autoid.Allocators) error { tblInfo, err := m.GetTable(dbInfo.ID, tableID) if err != nil { return errors.Trace(err) @@ -195,11 +195,10 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i ConvertCharsetCollateToLowerCaseIfNeed(tblInfo) ConvertOldVersionUTF8ToUTF8MB4IfNeed(tblInfo) - if alloc == nil { - schemaID := dbInfo.ID - alloc = autoid.NewAllocator(b.handle.store, tblInfo.GetDBID(schemaID), tblInfo.IsAutoIncColUnsigned()) + if len(allocs) == 0 { + allocs = autoid.NewAllocatorsFromTblInfo(b.handle.store, dbInfo.ID, tblInfo) } - tbl, err := tables.TableFromMeta(alloc, tblInfo) + tbl, err := tables.TableFromMeta(allocs, tblInfo) if err != nil { return errors.Trace(err) } @@ -331,7 +330,7 @@ func (b *Builder) InitWithDBInfos(dbInfos []*model.DBInfo, schemaVersion int64) return b, nil } -type tableFromMetaFunc func(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Table, error) +type tableFromMetaFunc func(alloc autoid.Allocators, tblInfo *model.TableInfo) (table.Table, error) func (b *Builder) createSchemaTablesForDB(di *model.DBInfo, tableFromMeta tableFromMetaFunc) error { schTbls := &schemaTables{ @@ -340,10 +339,9 @@ func (b *Builder) createSchemaTablesForDB(di *model.DBInfo, tableFromMeta tableF } b.is.schemaMap[di.Name.L] = schTbls for _, t := range di.Tables { - schemaID := di.ID - alloc := autoid.NewAllocator(b.handle.store, t.GetDBID(schemaID), t.IsAutoIncColUnsigned()) + allocs := autoid.NewAllocatorsFromTblInfo(b.handle.store, di.ID, t) var tbl table.Table - tbl, err := tableFromMeta(alloc, t) + tbl, err := tableFromMeta(allocs, t) if err != nil { return errors.Trace(err) } @@ -356,7 +354,7 @@ func (b *Builder) createSchemaTablesForDB(di *model.DBInfo, tableFromMeta tableF type virtualTableDriver struct { *model.DBInfo - TableFromMeta func(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Table, error) + TableFromMeta func(alloc autoid.Allocators, tblInfo *model.TableInfo) (table.Table, error) } var drivers []*virtualTableDriver diff --git a/infoschema/infoschema.go b/infoschema/infoschema.go index 9da6e2ee72dd0..266b2ac8b89b3 100644 --- a/infoschema/infoschema.go +++ b/infoschema/infoschema.go @@ -74,7 +74,7 @@ type InfoSchema interface { SchemaByID(id int64) (*model.DBInfo, bool) SchemaByTable(tableInfo *model.TableInfo) (*model.DBInfo, bool) TableByID(id int64) (table.Table, bool) - AllocByID(id int64) (autoid.Allocator, bool) + AllocByID(id int64) (autoid.Allocators, bool) AllSchemaNames() []string AllSchemas() []*model.DBInfo Clone() (result []*model.DBInfo) @@ -231,12 +231,12 @@ func (is *infoSchema) TableByID(id int64) (val table.Table, ok bool) { return slice[idx], true } -func (is *infoSchema) AllocByID(id int64) (autoid.Allocator, bool) { +func (is *infoSchema) AllocByID(id int64) (autoid.Allocators, bool) { tbl, ok := is.TableByID(id) if !ok { return nil, false } - return tbl.Allocator(nil), true + return tbl.AllAllocators(nil), true } func (is *infoSchema) AllSchemaNames() (names []string) { diff --git a/infoschema/metricschema/tables.go b/infoschema/metricschema/tables.go new file mode 100644 index 0000000000000..a6a76087ef092 --- /dev/null +++ b/infoschema/metricschema/tables.go @@ -0,0 +1,73 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package metricschema + +import ( + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/table" +) + +// metricSchemaTable stands for the fake table all its data is in the memory. +type metricSchemaTable struct { + infoschema.VirtualTable + meta *model.TableInfo + cols []*table.Column +} + +func tableFromMeta(alloc autoid.Allocators, meta *model.TableInfo) (table.Table, error) { + return createMetricSchemaTable(meta), nil +} + +// createMetricSchemaTable creates all metricSchemaTables +func createMetricSchemaTable(meta *model.TableInfo) *metricSchemaTable { + columns := make([]*table.Column, 0, len(meta.Columns)) + for _, colInfo := range meta.Columns { + col := table.ToColumn(colInfo) + columns = append(columns, col) + } + t := &metricSchemaTable{ + meta: meta, + cols: columns, + } + return t +} + +// Cols implements table.Table Type interface. +func (vt *metricSchemaTable) Cols() []*table.Column { + return vt.cols +} + +// WritableCols implements table.Table Type interface. +func (vt *metricSchemaTable) WritableCols() []*table.Column { + return vt.cols +} + +// GetID implements table.Table GetID interface. +func (vt *metricSchemaTable) GetPhysicalID() int64 { + return vt.meta.ID +} + +// Meta implements table.Table Type interface. +func (vt *metricSchemaTable) Meta() *model.TableInfo { + return vt.meta +} + +// IterRecords implements table.Table IterRecords interface. +func (vt *metricSchemaTable) IterRecords(_ sessionctx.Context, _ kv.Key, _ []*table.Column, _ table.RecordIterFunc) error { + return nil +} diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index 62970019bb7e8..e06d2b15e29c3 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -43,18 +43,18 @@ type perfSchemaTable struct { cols []*table.Column } -var pluginTable = make(map[string]func(autoid.Allocator, *model.TableInfo) (table.Table, error)) +var pluginTable = make(map[string]func(autoid.Allocators, *model.TableInfo) (table.Table, error)) // RegisterTable registers a new table into TiDB. func RegisterTable(tableName, sql string, - tableFromMeta func(autoid.Allocator, *model.TableInfo) (table.Table, error)) { + tableFromMeta func(autoid.Allocators, *model.TableInfo) (table.Table, error)) { perfSchemaTables = append(perfSchemaTables, sql) pluginTable[tableName] = tableFromMeta } -func tableFromMeta(alloc autoid.Allocator, meta *model.TableInfo) (table.Table, error) { +func tableFromMeta(allocs autoid.Allocators, meta *model.TableInfo) (table.Table, error) { if f, ok := pluginTable[meta.Name.L]; ok { - ret, err := f(alloc, meta) + ret, err := f(allocs, meta) return ret, err } return createPerfSchemaTable(meta), nil diff --git a/infoschema/tables.go b/infoschema/tables.go index 71a2701577eb9..45eb2eb8018d2 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -51,7 +51,7 @@ const ( catalogVal = "def" tableProfiling = "PROFILING" tablePartitions = "PARTITIONS" - tableKeyColumm = "KEY_COLUMN_USAGE" + tableKeyColumn = "KEY_COLUMN_USAGE" tableReferConst = "REFERENTIAL_CONSTRAINTS" tableSessionVar = "SESSION_VARIABLES" tablePlugins = "PLUGINS" @@ -1079,7 +1079,7 @@ func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *m if err != nil { return 0, err } - return tbl.Allocator(ctx).Base() + 1, nil + return tbl.Allocator(ctx, autoid.RowIDAllocType).Base() + 1, nil } func dataForViews(ctx sessionctx.Context, schemas []*model.DBInfo) ([][]types.Datum, error) { @@ -1729,7 +1729,7 @@ var tableNameToColumns = map[string][]columnInfo{ tableFiles: filesCols, tableProfiling: profilingCols, tablePartitions: partitionsCols, - tableKeyColumm: keyColumnUsageCols, + tableKeyColumn: keyColumnUsageCols, tableReferConst: referConstCols, tableSessionVar: sessionVarCols, tablePlugins: pluginsCols, @@ -1823,7 +1823,7 @@ func (it *infoschemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) fullRows = dataForPseudoProfiling() } case tablePartitions: - case tableKeyColumm: + case tableKeyColumn: fullRows = dataForKeyColumnUsage(dbs) case tableReferConst: case tablePlugins, tableTriggers: @@ -1977,7 +1977,12 @@ func (it *infoschemaTable) AllocHandle(ctx sessionctx.Context) (int64, error) { } // Allocator implements table.Table Allocator interface. -func (it *infoschemaTable) Allocator(ctx sessionctx.Context) autoid.Allocator { +func (it *infoschemaTable) Allocator(_ sessionctx.Context, _ autoid.AllocatorType) autoid.Allocator { + return nil +} + +// AllAllocators implements table.Table AllAllocators interface. +func (it *infoschemaTable) AllAllocators(_ sessionctx.Context) autoid.Allocators { return nil } @@ -2094,7 +2099,12 @@ func (vt *VirtualTable) AllocHandle(ctx sessionctx.Context) (int64, error) { } // Allocator implements table.Table Allocator interface. -func (vt *VirtualTable) Allocator(ctx sessionctx.Context) autoid.Allocator { +func (vt *VirtualTable) Allocator(_ sessionctx.Context, _ autoid.AllocatorType) autoid.Allocator { + return nil +} + +// AllAllocators implements table.Table AllAllocators interface. +func (vt *VirtualTable) AllAllocators(_ sessionctx.Context) autoid.Allocators { return nil } diff --git a/meta/autoid/autoid.go b/meta/autoid/autoid.go index 0b65c84595e13..e7974c6103173 100755 --- a/meta/autoid/autoid.go +++ b/meta/autoid/autoid.go @@ -23,7 +23,7 @@ import ( "github.com/cznic/mathutil" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/parser/terror" + "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/metrics" @@ -37,10 +37,26 @@ const ( defaultConsumeTime = 10 * time.Second ) +// RowIDBitLength is the bit number of a row id in TiDB. +const RowIDBitLength = 64 + +// DefaultAutoRandomBits is the default value of auto sharding. +const DefaultAutoRandomBits = 5 + // Test needs to change it, so it's a variable. var step = int64(30000) -var errInvalidTableID = terror.ClassAutoid.New(codeInvalidTableID, "invalid TableID") +// AllocatorType is the type of allocator for generating auto-id. Different type of allocators use different key-value pairs. +type AllocatorType = uint8 + +const ( + // RowIDAllocType indicates the allocator is used to allocate row id. + RowIDAllocType AllocatorType = iota + // AutoIncrementType indicates the allocator is used to allocate auto increment value. + AutoIncrementType + // AutoRandomType indicates the allocator is used to allocate auto-shard id. + AutoRandomType +) // Allocator is an auto increment id generator. // Just keep id unique actually. @@ -59,6 +75,15 @@ type Allocator interface { End() int64 // NextGlobalAutoID returns the next global autoID. NextGlobalAutoID(tableID int64) (int64, error) + GetType() AllocatorType +} + +// Allocators represents a set of `Allocator`s. +type Allocators []Allocator + +// NewAllocators packs multiple `Allocator`s into Allocators. +func NewAllocators(allocators ...Allocator) Allocators { + return allocators } type allocator struct { @@ -71,6 +96,7 @@ type allocator struct { isUnsigned bool lastAllocTime time.Time step int64 + allocType AllocatorType } // GetStep is only used by tests @@ -100,7 +126,7 @@ func (alloc *allocator) NextGlobalAutoID(tableID int64) (int64, error) { err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error { var err1 error m := meta.NewMeta(txn) - autoID, err1 = m.GetAutoTableID(alloc.dbID, tableID) + autoID, err1 = getAutoIDByAllocType(m, alloc.dbID, tableID, alloc.allocType) if err1 != nil { return errors.Trace(err1) } @@ -127,7 +153,7 @@ func (alloc *allocator) rebase4Unsigned(tableID int64, requiredBase uint64, allo startTime := time.Now() err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error { m := meta.NewMeta(txn) - currentEnd, err1 := m.GetAutoTableID(alloc.dbID, tableID) + currentEnd, err1 := getAutoIDByAllocType(m, alloc.dbID, tableID, alloc.allocType) if err1 != nil { return err1 } @@ -148,7 +174,7 @@ func (alloc *allocator) rebase4Unsigned(tableID int64, requiredBase uint64, allo newBase = requiredBase newEnd = requiredBase } - _, err1 = m.GenAutoTableID(alloc.dbID, tableID, int64(newEnd-uCurrentEnd)) + _, err1 = generateAutoIDByAllocType(m, alloc.dbID, tableID, int64(newEnd-uCurrentEnd), alloc.allocType) return err1 }) metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDRebase, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) @@ -173,7 +199,7 @@ func (alloc *allocator) rebase4Signed(tableID, requiredBase int64, allocIDs bool startTime := time.Now() err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error { m := meta.NewMeta(txn) - currentEnd, err1 := m.GetAutoTableID(alloc.dbID, tableID) + currentEnd, err1 := getAutoIDByAllocType(m, alloc.dbID, tableID, alloc.allocType) if err1 != nil { return err1 } @@ -193,7 +219,7 @@ func (alloc *allocator) rebase4Signed(tableID, requiredBase int64, allocIDs bool newBase = requiredBase newEnd = requiredBase } - _, err1 = m.GenAutoTableID(alloc.dbID, tableID, newEnd-currentEnd) + _, err1 = generateAutoIDByAllocType(m, alloc.dbID, tableID, newEnd-currentEnd, alloc.allocType) return err1 }) metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDRebase, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) @@ -221,6 +247,10 @@ func (alloc *allocator) Rebase(tableID, requiredBase int64, allocIDs bool) error return alloc.rebase4Signed(tableID, requiredBase, allocIDs) } +func (alloc *allocator) GetType() AllocatorType { + return alloc.allocType +} + // NextStep return new auto id step according to previous step and consuming time. func NextStep(curStep int64, consumeDur time.Duration) int64 { failpoint.Inject("mockAutoIDChange", func(val failpoint.Value) { @@ -240,18 +270,27 @@ func NextStep(curStep int64, consumeDur time.Duration) int64 { } // NewAllocator returns a new auto increment id generator on the store. -func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator { +func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool, allocType AllocatorType) Allocator { return &allocator{ store: store, dbID: dbID, isUnsigned: isUnsigned, step: step, lastAllocTime: time.Now(), + allocType: allocType, } } -//codeInvalidTableID is the code of autoid error. -const codeInvalidTableID terror.ErrCode = 1 +// NewAllocatorsFromTblInfo creates an array of allocators of different types with the information of model.TableInfo. +func NewAllocatorsFromTblInfo(store kv.Storage, schemaID int64, tblInfo *model.TableInfo) Allocators { + var allocs []Allocator + dbID := tblInfo.GetDBID(schemaID) + allocs = append(allocs, NewAllocator(store, dbID, tblInfo.IsAutoIncColUnsigned(), RowIDAllocType)) + if tblInfo.ContainsAutoRandomBits() { + allocs = append(allocs, NewAllocator(store, dbID, tblInfo.IsAutoRandomBitColUnsigned(), AutoRandomType)) + } + return NewAllocators(allocs...) +} var localSchemaID = int64(math.MaxInt64) @@ -298,7 +337,7 @@ func (alloc *allocator) alloc4Signed(tableID int64, n uint64) (int64, int64, err err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error { m := meta.NewMeta(txn) var err1 error - newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID) + newBase, err1 = getAutoIDByAllocType(m, alloc.dbID, tableID, alloc.allocType) if err1 != nil { return err1 } @@ -307,7 +346,7 @@ func (alloc *allocator) alloc4Signed(tableID int64, n uint64) (int64, int64, err if tmpStep < n1 { return ErrAutoincReadFailed } - newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep) + newEnd, err1 = generateAutoIDByAllocType(m, alloc.dbID, tableID, tmpStep, alloc.allocType) return err1 }) metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) @@ -352,7 +391,7 @@ func (alloc *allocator) alloc4Unsigned(tableID int64, n uint64) (int64, int64, e err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error { m := meta.NewMeta(txn) var err1 error - newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID) + newBase, err1 = getAutoIDByAllocType(m, alloc.dbID, tableID, alloc.allocType) if err1 != nil { return err1 } @@ -361,7 +400,7 @@ func (alloc *allocator) alloc4Unsigned(tableID int64, n uint64) (int64, int64, e if tmpStep < n1 { return ErrAutoincReadFailed } - newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep) + newEnd, err1 = generateAutoIDByAllocType(m, alloc.dbID, tableID, tmpStep, alloc.allocType) return err1 }) metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) @@ -384,3 +423,26 @@ func (alloc *allocator) alloc4Unsigned(tableID int64, n uint64) (int64, int64, e alloc.base = int64(uint64(alloc.base) + n) return min, alloc.base, nil } + +func getAutoIDByAllocType(m *meta.Meta, dbID, tableID int64, allocType AllocatorType) (int64, error) { + switch allocType { + // Currently, row id allocator and auto-increment value allocator shares the same key-value pair. + case RowIDAllocType, AutoIncrementType: + return m.GetAutoTableID(dbID, tableID) + case AutoRandomType: + return m.GetAutoRandomID(dbID, tableID) + default: + return 0, errInvalidAllocatorType.GenWithStackByArgs() + } +} + +func generateAutoIDByAllocType(m *meta.Meta, dbID, tableID, step int64, allocType AllocatorType) (int64, error) { + switch allocType { + case RowIDAllocType, AutoIncrementType: + return m.GenAutoTableID(dbID, tableID, step) + case AutoRandomType: + return m.GenAutoRandomID(dbID, tableID, step) + default: + return 0, errInvalidAllocatorType.GenWithStackByArgs() + } +} diff --git a/meta/autoid/autoid_test.go b/meta/autoid/autoid_test.go index 32ba1c49dbb98..2e96e4ab9fae6 100644 --- a/meta/autoid/autoid_test.go +++ b/meta/autoid/autoid_test.go @@ -67,7 +67,8 @@ func (*testSuite) TestT(c *C) { }) c.Assert(err, IsNil) - alloc := autoid.NewAllocator(store, 1, false) + // Since the test here is applicable to any type of allocators, autoid.RowIDAllocType is chosen. + alloc := autoid.NewAllocator(store, 1, false, autoid.RowIDAllocType) c.Assert(alloc, NotNil) globalAutoID, err := alloc.NextGlobalAutoID(1) @@ -107,13 +108,13 @@ func (*testSuite) TestT(c *C) { c.Assert(err, IsNil) c.Assert(id, Equals, int64(3011)) - alloc = autoid.NewAllocator(store, 1, false) + alloc = autoid.NewAllocator(store, 1, false, autoid.RowIDAllocType) c.Assert(alloc, NotNil) _, id, err = alloc.Alloc(1, 1) c.Assert(err, IsNil) c.Assert(id, Equals, int64(autoid.GetStep()+1)) - alloc = autoid.NewAllocator(store, 1, false) + alloc = autoid.NewAllocator(store, 1, false, autoid.RowIDAllocType) c.Assert(alloc, NotNil) err = alloc.Rebase(2, int64(1), false) c.Assert(err, IsNil) @@ -121,11 +122,11 @@ func (*testSuite) TestT(c *C) { c.Assert(err, IsNil) c.Assert(id, Equals, int64(2)) - alloc = autoid.NewAllocator(store, 1, false) + alloc = autoid.NewAllocator(store, 1, false, autoid.RowIDAllocType) c.Assert(alloc, NotNil) err = alloc.Rebase(3, int64(3210), false) c.Assert(err, IsNil) - alloc = autoid.NewAllocator(store, 1, false) + alloc = autoid.NewAllocator(store, 1, false, autoid.RowIDAllocType) c.Assert(alloc, NotNil) err = alloc.Rebase(3, int64(3000), false) c.Assert(err, IsNil) @@ -147,7 +148,7 @@ func (*testSuite) TestT(c *C) { c.Assert(err, IsNil) // alloc N for signed - alloc = autoid.NewAllocator(store, 1, false) + alloc = autoid.NewAllocator(store, 1, false, autoid.RowIDAllocType) c.Assert(alloc, NotNil) globalAutoID, err = alloc.NextGlobalAutoID(4) c.Assert(err, IsNil) @@ -216,7 +217,7 @@ func (*testSuite) TestUnsignedAutoid(c *C) { }) c.Assert(err, IsNil) - alloc := autoid.NewAllocator(store, 1, true) + alloc := autoid.NewAllocator(store, 1, true, autoid.RowIDAllocType) c.Assert(alloc, NotNil) globalAutoID, err := alloc.NextGlobalAutoID(1) @@ -256,13 +257,13 @@ func (*testSuite) TestUnsignedAutoid(c *C) { c.Assert(err, IsNil) c.Assert(id, Equals, int64(3011)) - alloc = autoid.NewAllocator(store, 1, true) + alloc = autoid.NewAllocator(store, 1, true, autoid.RowIDAllocType) c.Assert(alloc, NotNil) _, id, err = alloc.Alloc(1, 1) c.Assert(err, IsNil) c.Assert(id, Equals, int64(autoid.GetStep()+1)) - alloc = autoid.NewAllocator(store, 1, true) + alloc = autoid.NewAllocator(store, 1, true, autoid.RowIDAllocType) c.Assert(alloc, NotNil) err = alloc.Rebase(2, int64(1), false) c.Assert(err, IsNil) @@ -270,11 +271,11 @@ func (*testSuite) TestUnsignedAutoid(c *C) { c.Assert(err, IsNil) c.Assert(id, Equals, int64(2)) - alloc = autoid.NewAllocator(store, 1, true) + alloc = autoid.NewAllocator(store, 1, true, autoid.RowIDAllocType) c.Assert(alloc, NotNil) err = alloc.Rebase(3, int64(3210), false) c.Assert(err, IsNil) - alloc = autoid.NewAllocator(store, 1, true) + alloc = autoid.NewAllocator(store, 1, true, autoid.RowIDAllocType) c.Assert(alloc, NotNil) err = alloc.Rebase(3, int64(3000), false) c.Assert(err, IsNil) @@ -299,7 +300,7 @@ func (*testSuite) TestUnsignedAutoid(c *C) { c.Assert(err, IsNil) // alloc N for unsigned - alloc = autoid.NewAllocator(store, 1, true) + alloc = autoid.NewAllocator(store, 1, true, autoid.RowIDAllocType) c.Assert(alloc, NotNil) globalAutoID, err = alloc.NextGlobalAutoID(4) c.Assert(err, IsNil) @@ -329,7 +330,7 @@ func (*testSuite) TestUnsignedAutoid(c *C) { } // TestConcurrentAlloc is used for the test that -// multiple alloctors allocate ID with the same table ID concurrently. +// multiple allocators allocate ID with the same table ID concurrently. func (*testSuite) TestConcurrentAlloc(c *C) { store, err := mockstore.NewMockTikvStore() c.Assert(err, IsNil) @@ -358,7 +359,7 @@ func (*testSuite) TestConcurrentAlloc(c *C) { errCh := make(chan error, count) allocIDs := func() { - alloc := autoid.NewAllocator(store, dbID, false) + alloc := autoid.NewAllocator(store, dbID, false, autoid.RowIDAllocType) for j := 0; j < int(autoid.GetStep())+5; j++ { _, id, err1 := alloc.Alloc(tblID, 1) if err1 != nil { @@ -436,7 +437,7 @@ func (*testSuite) TestRollbackAlloc(c *C) { injectConf := new(kv.InjectionConfig) injectConf.SetCommitError(errors.New("injected")) injectedStore := kv.NewInjectedStore(store, injectConf) - alloc := autoid.NewAllocator(injectedStore, 1, false) + alloc := autoid.NewAllocator(injectedStore, 1, false, autoid.RowIDAllocType) _, _, err = alloc.Alloc(2, 1) c.Assert(err, NotNil) c.Assert(alloc.Base(), Equals, int64(0)) @@ -482,7 +483,7 @@ func BenchmarkAllocator_Alloc(b *testing.B) { if err != nil { return } - alloc := autoid.NewAllocator(store, 1, false) + alloc := autoid.NewAllocator(store, 1, false, autoid.RowIDAllocType) b.StartTimer() for i := 0; i < b.N; i++ { alloc.Alloc(2, 1) diff --git a/meta/autoid/errors.go b/meta/autoid/errors.go index e9b0b0fa6fae4..58441832f5919 100644 --- a/meta/autoid/errors.go +++ b/meta/autoid/errors.go @@ -20,10 +20,16 @@ import ( // Error instances. var ( - ErrAutoincReadFailed = terror.ClassAutoid.New(mysql.ErrAutoincReadFailed, mysql.MySQLErrName[mysql.ErrAutoincReadFailed]) - ErrWrongAutoKey = terror.ClassAutoid.New(mysql.ErrWrongAutoKey, mysql.MySQLErrName[mysql.ErrWrongAutoKey]) + ErrAutoincReadFailed = terror.ClassAutoid.New(mysql.ErrAutoincReadFailed, mysql.MySQLErrName[mysql.ErrAutoincReadFailed]) + ErrWrongAutoKey = terror.ClassAutoid.New(mysql.ErrWrongAutoKey, mysql.MySQLErrName[mysql.ErrWrongAutoKey]) + errInvalidTableID = terror.ClassAutoid.New(codeInvalidTableID, "invalid TableID") + errInvalidAllocatorType = terror.ClassAutoid.New(mysql.ErrUnknownAllocatorType, mysql.MySQLErrName[mysql.ErrUnknownAllocatorType]) + ErrAutoRandReadFailed = terror.ClassAutoid.New(mysql.ErrAutoRandReadFailed, mysql.MySQLErrName[mysql.ErrAutoRandReadFailed]) ) +// codeInvalidTableID is the code of autoid error. +const codeInvalidTableID terror.ErrCode = 8056 + func init() { // Map error codes to mysql error codes. tableMySQLErrCodes := map[terror.ErrCode]uint16{ @@ -32,3 +38,22 @@ func init() { } terror.ErrClassToMySQLCodes[terror.ClassAutoid] = tableMySQLErrCodes } + +const ( + // AutoRandomPKisNotHandleErrMsg indicates the auto_random column attribute is defined on a non-primary key column, or the table's primary key is not a single integer column. + AutoRandomPKisNotHandleErrMsg = "column %s is not the single integer primary key, or alter-primary-key is enabled" + // AutoRandomExperimentalDisabledErrMsg is reported when the experimental option allow-auto-random is not enabled. + AutoRandomExperimentalDisabledErrMsg = "auto_random is an experimental feature, which can only be used when allow-auto-random is enabled. This can be changed in the configuration." + // AutoRandomIncompatibleWithAutoIncErrMsg is reported when auto_random and auto_increment are specified on the same column. + AutoRandomIncompatibleWithAutoIncErrMsg = "auto_random is incompatible with auto_increment" + // AutoRandomIncompatibleWithDefaultValueErrMsg is reported when auto_random and default are specified on the same column. + AutoRandomIncompatibleWithDefaultValueErrMsg = "auto_random is incompatible with default" + // AutoRandomOverflowErrMsg is reported when auto_random is greater than max length of a MySQL data type. + AutoRandomOverflowErrMsg = "auto_random = %d will overflow. The max length of bits is %d" + // AutoRandomModifyColTypeErrMsg is reported when a user is trying to modify the type of a column specified with auto_random. + AutoRandomModifyColTypeErrMsg = "modifying the auto_random column type is not supported" + // AutoRandomAlterErrMsg is reported when a user is trying to add/drop/modify the value of auto_random attribute. + AutoRandomAlterErrMsg = "adding/dropping/modifying auto_random is not supported" + // AutoRandomNonPositive is reported then a user specifies a non-positive value for auto_random. + AutoRandomNonPositive = "the value of auto_random should be positive" +) diff --git a/meta/meta.go b/meta/meta.go index 4bee95771c172..bc9f3e9b5ecac 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -63,6 +63,7 @@ var ( mDBPrefix = "DB" mTablePrefix = "Table" mTableIDPrefix = "TID" + mRandomIDPrefix = "TARID" mBootstrapKey = []byte("BootstrapKey") mSchemaDiffPrefix = "Diff" ) @@ -131,6 +132,10 @@ func (m *Meta) autoTableIDKey(tableID int64) []byte { return []byte(fmt.Sprintf("%s:%d", mTableIDPrefix, tableID)) } +func (m *Meta) autoRandomTableIDKey(tableID int64) []byte { + return []byte(fmt.Sprintf("%s:%d", mRandomIDPrefix, tableID)) +} + func (m *Meta) tableKey(tableID int64) []byte { return []byte(fmt.Sprintf("%s:%d", mTablePrefix, tableID)) } @@ -163,11 +168,32 @@ func (m *Meta) GenAutoTableID(dbID, tableID, step int64) (int64, error) { return m.txn.HInc(dbKey, m.autoTableIDKey(tableID), step) } +// GenAutoRandomID adds step to the auto shard ID of the table and returns the sum. +func (m *Meta) GenAutoRandomID(dbID, tableID, step int64) (int64, error) { + // Check if DB exists. + dbKey := m.dbKey(dbID) + if err := m.checkDBExists(dbKey); err != nil { + return 0, errors.Trace(err) + } + // Check if table exists. + tableKey := m.tableKey(tableID) + if err := m.checkTableExists(dbKey, tableKey); err != nil { + return 0, errors.Trace(err) + } + + return m.txn.HInc(dbKey, m.autoRandomTableIDKey(tableID), step) +} + // GetAutoTableID gets current auto id with table id. func (m *Meta) GetAutoTableID(dbID int64, tableID int64) (int64, error) { return m.txn.HGetInt64(m.dbKey(dbID), m.autoTableIDKey(tableID)) } +// GetAutoRandomID gets current auto shard id with table id. +func (m *Meta) GetAutoRandomID(dbID int64, tableID int64) (int64, error) { + return m.txn.HGetInt64(m.dbKey(dbID), m.autoRandomTableIDKey(tableID)) +} + // GetSchemaVersion gets current global schema version. func (m *Meta) GetSchemaVersion() (int64, error) { return m.txn.GetInt64(mSchemaVersionKey) @@ -272,7 +298,16 @@ func (m *Meta) CreateTableAndSetAutoID(dbID int64, tableInfo *model.TableInfo, a return errors.Trace(err) } _, err = m.txn.HInc(m.dbKey(dbID), m.autoTableIDKey(tableInfo.ID), autoID) - return errors.Trace(err) + if err != nil { + return errors.Trace(err) + } + if tableInfo.AutoRandomBits > 0 { + _, err = m.txn.HInc(m.dbKey(dbID), m.autoRandomTableIDKey(tableInfo.ID), 0) + if err != nil { + return errors.Trace(err) + } + } + return nil } // DropDatabase drops whole database. @@ -313,6 +348,9 @@ func (m *Meta) DropTableOrView(dbID int64, tblID int64, delAutoID bool) error { if err := m.txn.HDel(dbKey, m.autoTableIDKey(tblID)); err != nil { return errors.Trace(err) } + if err := m.txn.HDel(dbKey, m.autoRandomTableIDKey(tblID)); err != nil { + return errors.Trace(err) + } } return nil } diff --git a/sessionctx/binloginfo/binloginfo.go b/sessionctx/binloginfo/binloginfo.go index 638da8dd42b73..f7db265e6f340 100644 --- a/sessionctx/binloginfo/binloginfo.go +++ b/sessionctx/binloginfo/binloginfo.go @@ -15,6 +15,7 @@ package binloginfo import ( "context" + "fmt" "math" "regexp" "strings" @@ -23,6 +24,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/parser" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb-tools/tidb-binlog/node" pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client" @@ -46,6 +48,7 @@ var pumpsClient *pumpcli.PumpsClient var pumpsClientLock sync.RWMutex var shardPat = regexp.MustCompile(`SHARD_ROW_ID_BITS\s*=\s*\d+\s*`) var preSplitPat = regexp.MustCompile(`PRE_SPLIT_REGIONS\s*=\s*\d+\s*`) +var autoRandomPat = regexp.MustCompile(`AUTO_RANDOM\s*\(\s*\d+\s*\)\s*`) // BinlogInfo contains binlog data and binlog client. type BinlogInfo struct { @@ -298,18 +301,21 @@ func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, } const specialPrefix = `/*!90000 ` +const specialVersionPrefix = `/*T!%s ` // AddSpecialComment uses to add comment for table option in DDL query. // Export for testing. func AddSpecialComment(ddlQuery string) string { - if strings.Contains(ddlQuery, specialPrefix) { + if strings.Contains(ddlQuery, specialPrefix) || parser.SpecVersionCodePattern.MatchString(ddlQuery) { return ddlQuery } - return addSpecialCommentByRegexps(ddlQuery, shardPat, preSplitPat) + ddlQuery = addSpecialCommentByRegexps(ddlQuery, specialPrefix, shardPat, preSplitPat) + ddlQuery = addSpecialCommentByRegexps(ddlQuery, fmt.Sprintf(specialVersionPrefix, parser.CommentCodeAutoRandom), autoRandomPat) + return ddlQuery } // addSpecialCommentByRegexps uses to add special comment for the worlds in the ddlQuery with match the regexps. -func addSpecialCommentByRegexps(ddlQuery string, regs ...*regexp.Regexp) string { +func addSpecialCommentByRegexps(ddlQuery string, prefix string, regs ...*regexp.Regexp) string { upperQuery := strings.ToUpper(ddlQuery) var specialComments []string minIdx := math.MaxInt64 @@ -328,7 +334,7 @@ func addSpecialCommentByRegexps(ddlQuery string, regs ...*regexp.Regexp) string upperQuery = upperQuery[:loc[0]] + upperQuery[loc[1]:] } if minIdx != math.MaxInt64 { - query := ddlQuery[:minIdx] + specialPrefix + query := ddlQuery[:minIdx] + prefix for _, comment := range specialComments { if query[len(query)-1] != ' ' { query += " " diff --git a/sessionctx/binloginfo/binloginfo_test.go b/sessionctx/binloginfo/binloginfo_test.go index dead446276f9a..e86987646f8c8 100644 --- a/sessionctx/binloginfo/binloginfo_test.go +++ b/sessionctx/binloginfo/binloginfo_test.go @@ -467,6 +467,18 @@ func (s *testBinlogSuite) TestAddSpecialComment(c *C) { "alter table t shard_row_id_bits=2 ", "alter table t /*!90000 shard_row_id_bits=2 */", }, + { + "create table t1 (id int primary key auto_random(2));", + "create table t1 (id int primary key /*T!40000 auto_random(2) */ );", + }, + { + "create table t1 (id int auto_random ( 4 ) primary key);", + "create table t1 (id int /*T!40000 auto_random ( 4 ) */ primary key);", + }, + { + "create table t1 (id int auto_random ( 4 ) primary key);", + "create table t1 (id int /*T!40000 auto_random ( 4 ) */ primary key);", + }, } for _, ca := range testCase { re := binloginfo.AddSpecialComment(ca.input) diff --git a/table/table.go b/table/table.go index c06fa60186333..a7c41185a64c8 100644 --- a/table/table.go +++ b/table/table.go @@ -150,7 +150,10 @@ type Table interface { AllocHandle(ctx sessionctx.Context) (int64, error) // Allocator returns Allocator. - Allocator(ctx sessionctx.Context) autoid.Allocator + Allocator(ctx sessionctx.Context, allocatorType autoid.AllocatorType) autoid.Allocator + + // AllAllocators returns all allocators. + AllAllocators(ctx sessionctx.Context) autoid.Allocators // RebaseAutoID rebases the auto_increment ID base. // If allocIDs is true, it will allocate some IDs and save to the cache. @@ -173,7 +176,7 @@ func AllocAutoIncrementValue(ctx context.Context, t Table, sctx sessionctx.Conte span1 := span.Tracer().StartSpan("table.AllocAutoIncrementValue", opentracing.ChildOf(span.Context())) defer span1.Finish() } - _, max, err := t.Allocator(sctx).Alloc(t.Meta().ID, uint64(1)) + _, max, err := t.Allocator(sctx, autoid.RowIDAllocType).Alloc(t.Meta().ID, uint64(1)) if err != nil { return 0, err } @@ -186,7 +189,7 @@ func AllocBatchAutoIncrementValue(ctx context.Context, t Table, sctx sessionctx. span1 := span.Tracer().StartSpan("table.AllocBatchAutoIncrementValue", opentracing.ChildOf(span.Context())) defer span1.Finish() } - return t.Allocator(sctx).Alloc(t.Meta().ID, uint64(N)) + return t.Allocator(sctx, autoid.RowIDAllocType).Alloc(t.Meta().ID, uint64(N)) } // PhysicalTable is an abstraction for two kinds of table representation: partition or non-partitioned table. @@ -207,7 +210,7 @@ type PartitionedTable interface { // TableFromMeta builds a table.Table from *model.TableInfo. // Currently, it is assigned to tables.TableFromMeta in tidb package's init function. -var TableFromMeta func(alloc autoid.Allocator, tblInfo *model.TableInfo) (Table, error) +var TableFromMeta func(allocators autoid.Allocators, tblInfo *model.TableInfo) (Table, error) // MockTableFromMeta only serves for test. var MockTableFromMeta func(tableInfo *model.TableInfo) Table diff --git a/table/tables/partition.go b/table/tables/partition.go index 58e24385a4c2f..b1f5867133c91 100644 --- a/table/tables/partition.go +++ b/table/tables/partition.go @@ -79,7 +79,7 @@ func newPartitionedTable(tbl *Table, tblInfo *model.TableInfo) (table.Table, err pi := tblInfo.GetPartitionInfo() for _, p := range pi.Definitions { var t partition - err := initTableCommonWithIndices(&t.tableCommon, tblInfo, p.ID, tbl.Columns, tbl.alloc) + err := initTableCommonWithIndices(&t.tableCommon, tblInfo, p.ID, tbl.Columns, tbl.allocs) if err != nil { return nil, errors.Trace(err) } diff --git a/table/tables/tables.go b/table/tables/tables.go index 3778ed5453d14..30bbb53129a1f 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -53,7 +53,7 @@ type tableCommon struct { writableIndices []table.Index indices []table.Index meta *model.TableInfo - alloc autoid.Allocator + allocs autoid.Allocators // recordPrefix and indexPrefix are generated using physicalTableID. recordPrefix kv.Key @@ -92,7 +92,7 @@ func MockTableFromMeta(tblInfo *model.TableInfo) table.Table { } // TableFromMeta creates a Table instance from model.TableInfo. -func TableFromMeta(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Table, error) { +func TableFromMeta(allocs autoid.Allocators, tblInfo *model.TableInfo) (table.Table, error) { if tblInfo.State == model.StateNone { return nil, table.ErrTableStateCantNone.GenWithStack("table %s can't be in none state", tblInfo.Name) } @@ -125,7 +125,7 @@ func TableFromMeta(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Tabl } var t Table - initTableCommon(&t.tableCommon, tblInfo, tblInfo.ID, columns, alloc) + initTableCommon(&t.tableCommon, tblInfo, tblInfo.ID, columns, allocs) if tblInfo.GetPartitionInfo() == nil { if err := initTableIndices(&t.tableCommon); err != nil { return nil, err @@ -137,10 +137,10 @@ func TableFromMeta(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Tabl } // initTableCommon initializes a tableCommon struct. -func initTableCommon(t *tableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, alloc autoid.Allocator) { +func initTableCommon(t *tableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, allocs autoid.Allocators) { t.tableID = tblInfo.ID t.physicalTableID = physicalTableID - t.alloc = alloc + t.allocs = allocs t.meta = tblInfo t.Columns = cols t.publicColumns = t.Cols() @@ -165,8 +165,8 @@ func initTableIndices(t *tableCommon) error { return nil } -func initTableCommonWithIndices(t *tableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, alloc autoid.Allocator) error { - initTableCommon(t, tblInfo, physicalTableID, cols, alloc) +func initTableCommonWithIndices(t *tableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, allocs autoid.Allocators) error { + initTableCommon(t, tblInfo, physicalTableID, cols, allocs) return initTableIndices(t) } @@ -434,12 +434,10 @@ func (t *tableCommon) AddRecord(ctx sessionctx.Context, r []types.Datum, opts .. recordID = r[len(r)-1].GetInt64() hasRecordID = true } else { - for _, col := range cols { - if col.IsPKHandleColumn(t.meta) { - recordID = r[col.Offset].GetInt64() - hasRecordID = true - break - } + tblInfo := t.Meta() + if tblInfo.PKIsHandle { + recordID = r[tblInfo.GetPkColInfo().Offset].GetInt64() + hasRecordID = true } } if !hasRecordID { @@ -912,17 +910,17 @@ func GetColDefaultValue(ctx sessionctx.Context, col *table.Column, defaultVals [ // AllocHandle implements table.Table AllocHandle interface. func (t *tableCommon) AllocHandle(ctx sessionctx.Context) (int64, error) { - _, rowID, err := t.Allocator(ctx).Alloc(t.tableID, 1) + _, rowID, err := t.Allocator(ctx, autoid.RowIDAllocType).Alloc(t.tableID, 1) if err != nil { return 0, err } if t.meta.ShardRowIDBits > 0 { // Use max record ShardRowIDBits to check overflow. - if OverflowShardBits(rowID, t.meta.MaxShardRowIDBits) { + if OverflowShardBits(rowID, t.meta.MaxShardRowIDBits, autoid.RowIDBitLength) { // If overflow, the rowID may be duplicated. For examples, // t.meta.ShardRowIDBits = 4 // rowID = 0010111111111111111111111111111111111111111111111111111111111111 - // shard = 01000000000000000000000000000000000000000000000000000000000000000 + // shard = 0100000000000000000000000000000000000000000000000000000000000000 // will be duplicated with: // rowID = 0100111111111111111111111111111111111111111111111111111111111111 // shard = 0010000000000000000000000000000000000000000000000000000000000000 @@ -930,7 +928,7 @@ func (t *tableCommon) AllocHandle(ctx sessionctx.Context) (int64, error) { } txnCtx := ctx.GetSessionVars().TxnCtx if txnCtx.Shard == nil { - shard := t.calcShard(txnCtx.StartTS) + shard := CalcShard(t.meta.ShardRowIDBits, txnCtx.StartTS, autoid.RowIDBitLength) txnCtx.Shard = &shard } rowID |= *txnCtx.Shard @@ -938,33 +936,59 @@ func (t *tableCommon) AllocHandle(ctx sessionctx.Context) (int64, error) { return rowID, nil } -// OverflowShardBits checks whether the rowID overflow `1<<(64-shardRowIDBits-1) -1`. -func OverflowShardBits(rowID int64, shardRowIDBits uint64) bool { - mask := (1< 0 +// OverflowShardBits checks whether the recordID overflow `1<<(typeBitsLength-shardRowIDBits-1) -1`. +func OverflowShardBits(recordID int64, shardRowIDBits uint64, typeBitsLength uint64) bool { + mask := (1< 0 } -func (t *tableCommon) calcShard(startTS uint64) int64 { +// CalcShard calculates the shard prefix by hashing the startTS. Make sure OverflowShardBits is false before calling it. +func CalcShard(shardRowIDBits uint64, startTS uint64, typeBitsLength uint64) int64 { var buf [8]byte binary.LittleEndian.PutUint64(buf[:], startTS) hashVal := int64(murmur3.Sum32(buf[:])) - return (hashVal & (1<