Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl,mdl: fix the issue that the table_ids overlap regex is not correct (#52879) #53191

Closed
wants to merge 1 commit into from
+664 −1
Closed

ddl,mdl: fix the issue that the table_ids overlap regex is not correct (#52879) #53191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions br/pkg/restore/db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,3 +415,17 @@ func TestGetExistedUserDBs(t *testing.T) {
dbs = restore.GetExistedUserDBs(dom)
require.Equal(t, 2, len(dbs))
}
<<<<<<< HEAD
=======

// NOTICE: Once there is a new system table, BR needs to ensure that it is correctly classified:
//
// - IF it is an unrecoverable table, please add the table name into `unRecoverableTable`.
// - IF it is an system privilege table, please add the table name into `sysPrivilegeTableMap`.
// - IF it is an statistics table, please add the table name into `statsTables`.
//
// The above variables are in the file br/pkg/restore/systable_restore.go
func TestMonitorTheSystemTableIncremental(t *testing.T) {
require.Equal(t, int64(197), session.CurrentBootstrapVersion)
}
>>>>>>> af825ac7d8b (ddl,mdl: fix the issue that the table_ids overlap regex is not correct (#52879))
2 changes: 1 addition & 1 deletion ddl/job_table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func (d *ddl) getGeneralJob(sess *sess.Session) (*model.Job, error) {
}
// Check if there is any running job works on the same table.
sql := fmt.Sprintf("select job_id from mysql.tidb_ddl_job t1, (select table_ids from mysql.tidb_ddl_job where job_id = %d) t2 where "+
"(processing and CONCAT(',', t2.table_ids, ',') REGEXP CONCAT(',', REPLACE(t1.table_ids, ',', '|'), ',') != 0)"+
"(processing and CONCAT(',', t2.table_ids, ',') REGEXP CONCAT(',(', REPLACE(t1.table_ids, ',', '|'), '),') != 0)"+
"or (type = %d and processing)", job.ID, model.ActionFlashbackCluster)
rows, err := sess.Execute(d.ctx, sql, "check conflict jobs")
return len(rows) == 0, err
Expand Down
256 changes: 256 additions & 0 deletions infoschema/cluster_tables_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/pingcap/fn"
"github.com/pingcap/kvproto/pkg/deadlock"
"github.com/pingcap/kvproto/pkg/metapb"
<<<<<<< HEAD:infoschema/cluster_tables_test.go
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/executor"
Expand All @@ -53,6 +54,31 @@ import (
"github.com/pingcap/tidb/util/resourcegrouptag"
"github.com/pingcap/tidb/util/set"
"github.com/pingcap/tidb/util/stmtsummary"
=======
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/executor"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/infoschema/internal"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser"
"github.com/pingcap/tidb/pkg/parser/auth"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/privilege/privileges"
"github.com/pingcap/tidb/pkg/server"
"github.com/pingcap/tidb/pkg/store/mockstore"
"github.com/pingcap/tidb/pkg/store/mockstore/mockstorage"
"github.com/pingcap/tidb/pkg/store/mockstore/unistore"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/resourcegrouptag"
"github.com/pingcap/tidb/pkg/util/set"
"github.com/pingcap/tidb/pkg/util/stmtsummary"
>>>>>>> af825ac7d8b (ddl,mdl: fix the issue that the table_ids overlap regex is not correct (#52879)):pkg/infoschema/test/clustertablestest/cluster_tables_test.go
"github.com/pingcap/tipb/go-tipb"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/testutils"
Expand Down Expand Up @@ -1467,3 +1493,233 @@ func TestCreateBindingForPrepareToken(t *testing.T) {
tk.MustExec(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]))
}
}
<<<<<<< HEAD:infoschema/cluster_tables_test.go
=======

func testIndexUsageTable(t *testing.T, clusterTable bool) {
var tk *testkit.TestKit
var tableName string

if clusterTable {
s := new(clusterTablesSuite)
s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
s.startTime = time.Now()
defer s.httpServer.Close()
defer s.rpcserver.Stop()
tk = s.newTestKitWithRoot(t)
tableName = infoschema.ClusterTableTiDBIndexUsage
} else {
store := testkit.CreateMockStore(t)
tk = testkit.NewTestKit(t, store)
tableName = infoschema.TableTiDBIndexUsage
}

tk.MustExec("use test")
tk.MustExec("create table t1(id1 int unique, id2 int unique)")
tk.MustExec("create table t2(id1 int unique, id2 int unique)")

for i := 0; i < 100; i++ {
for j := 1; j <= 2; j++ {
tk.MustExec(fmt.Sprintf("insert into t%d values (?, ?)", j), i, i)
}
}
tk.MustExec("analyze table t1, t2")
tk.RefreshSession()
tk.MustExec("use test")
// range scan 0-10 through t1 id1
tk.MustQuery("select * from t1 use index(id1) where id1 >= 0 and id1 < 10")
// range scan 10-30 through t1 id2
tk.MustQuery("select * from t1 use index(id2) where id2 >= 10 and id2 < 30")
// range scan 30-60 through t2 id1
tk.MustQuery("select * from t2 use index(id1) where id1 >= 30 and id1 < 60")
// range scan 60-100 through t2 id2
tk.MustQuery("select * from t2 use index(id2) where id2 >= 60 and id2 < 100")
tk.RefreshSession()

require.Eventually(t, func() bool {
result := tk.MustQuery(fmt.Sprintf(`select
query_total,
rows_access_total,
percentage_access_0,
percentage_access_0_1,
percentage_access_1_10,
percentage_access_10_20,
percentage_access_20_50,
percentage_access_50_100,
percentage_access_100
from information_schema.%s
where table_schema='test' and
(table_name='t1' or table_name='t2') and
(index_name='id1' or index_name='id2') and
last_access_time is not null
order by table_name, index_name;`, tableName))
expectedResult := testkit.Rows(
"1 10 0 0 0 1 0 0 0",
"1 20 0 0 0 0 1 0 0",
"1 30 0 0 0 0 1 0 0",
"1 40 0 0 0 0 1 0 0")
if !result.Equal(expectedResult) {
logutil.BgLogger().Warn("result not equal", zap.Any("rows", result.Rows()))
return false
}
return true
}, time.Second*5, time.Millisecond*100)

// use another less-privileged user to select
tk.MustExec("create user test_user")
tk.MustExec("grant all privileges on test.t1 to test_user")
tk.RefreshSession()
require.NoError(t, tk.Session().Auth(&auth.UserIdentity{
Username: "test_user",
Hostname: "127.0.0.1",
}, nil, nil, nil))
// `test_user` cannot see table `t2`.
tk.MustQuery(fmt.Sprintf(`select
query_total,
rows_access_total,
percentage_access_0,
percentage_access_0_1,
percentage_access_1_10,
percentage_access_10_20,
percentage_access_20_50,
percentage_access_50_100,
percentage_access_100
from information_schema.%s
where table_schema='test' and
(table_name='t1' or table_name='t2') and
(index_name='id1' or index_name='id2') and
last_access_time is not null
order by table_name, index_name;`, tableName)).Check(testkit.Rows(
"1 10 0 0 0 1 0 0 0",
"1 20 0 0 0 0 1 0 0"))
}

func TestIndexUsageTable(t *testing.T) {
testIndexUsageTable(t, false)
}

func TestClusterIndexUsageTable(t *testing.T) {
testIndexUsageTable(t, true)
}

func TestUnusedIndexView(t *testing.T) {
s := new(clusterTablesSuite)
s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
s.startTime = time.Now()
defer s.httpServer.Close()
defer s.rpcserver.Stop()
tk := s.newTestKitWithRoot(t)

tk.MustExec("use test")
tk.MustExec("create table t(id1 int unique, id2 int unique)")
for i := 0; i < 100; i++ {
tk.MustExec("insert into t values (?, ?)", i, i)
}
tk.MustExec("analyze table t")
tk.RefreshSession()
tk.MustExec("use test")
// range scan 0-10 through t1 id1
tk.MustQuery("select * from t use index(id1) where id1 >= 0 and id1 < 10")
tk.MustHavePlan("select * from t use index(id1) where id1 >= 0 and id1 < 10", "IndexLookUp")
tk.RefreshSession()
// the index `id2` is unused
require.Eventually(t, func() bool {
result := tk.MustQuery(`select * from sys.schema_unused_indexes where object_name = 't'`)
logutil.BgLogger().Info("select schema_unused_indexes", zap.Any("row", result.Rows()))
expectedResult := testkit.Rows("test t id2")
return result.Equal(expectedResult)
}, 5*time.Second, 100*time.Millisecond)
}

func TestMDLViewIDConflict(t *testing.T) {
save := privileges.SkipWithGrant
privileges.SkipWithGrant = true
defer func() {
privileges.SkipWithGrant = save
}()

s := new(clusterTablesSuite)
s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
s.startTime = time.Now()
defer s.httpServer.Close()
tk := s.newTestKitWithRoot(t)

tk.MustExec("use test")
tk.MustExec("create table t(a int);")
tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
tk.MustExec("insert into t values (1)")

bigID := tbl.Meta().ID * 10
bigTableName := ""
// set a hard limitation on 10000 to avoid using too much resource
for i := 0; i < 10000; i++ {
bigTableName = fmt.Sprintf("t%d", i)
tk.MustExec(fmt.Sprintf("create table %s(a int);", bigTableName))

tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr(bigTableName))
require.NoError(t, err)

require.LessOrEqual(t, tbl.Meta().ID, bigID)
if tbl.Meta().ID == bigID {
break
}
}
tk.MustExec("insert into t1 values (1)")
tk.MustExec(fmt.Sprintf("insert into %s values (1)", bigTableName))

// Now we have two table: t and `bigTableName`. The later one's ID is 10 times the former one.
// Then create two session to run TXNs on these two tables
txnTK1 := s.newTestKitWithRoot(t)
txnTK2 := s.newTestKitWithRoot(t)
txnTK1.MustExec("use test")
txnTK1.MustExec("BEGIN")
// this transaction will query `t` and one another table. Then the `related_table_ids` is `smallID|anotherID`
txnTK1.MustQuery("SELECT * FROM t").Check(testkit.Rows("1"))
txnTK1.MustQuery("SELECT * FROM t1").Check(testkit.Rows("1"))
txnTK2.MustExec("use test")
txnTK2.MustExec("BEGIN")
txnTK2.MustQuery("SELECT * FROM " + bigTableName).Check(testkit.Rows("1"))

testTK := s.newTestKitWithRoot(t)
s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", testTK.Session().GetSessionManager())
defer s.rpcserver.Stop()
testTK.MustQuery("select table_name from mysql.tidb_mdl_view").Check(testkit.Rows())

// run a DDL on the table with smallID
ddlTK1 := s.newTestKitWithRoot(t)
ddlTK1.MustExec("use test")
wg := &sync.WaitGroup{}
wg.Add(1)
go func() {
ddlTK1.MustExec("ALTER TABLE t ADD COLUMN b INT;")
wg.Done()
}()
ddlTK2 := s.newTestKitWithRoot(t)
ddlTK2.MustExec("use test")
wg.Add(1)
go func() {
ddlTK2.MustExec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN b INT;", bigTableName))
wg.Done()
}()

require.Eventually(t, func() bool {
rows := testTK.MustQuery("select table_ids from mysql.tidb_mdl_info").Rows()
return len(rows) == 2
}, time.Second*10, time.Second)

// it only contains the table with smallID
require.Eventually(t, func() bool {
rows := testTK.MustQuery("select table_name, query, start_time from mysql.tidb_mdl_view order by table_name").Rows()
return len(rows) == 2
}, time.Second*10, time.Second)
txnTK1.MustExec("COMMIT")
txnTK2.MustExec("COMMIT")
wg.Wait()
}
>>>>>>> af825ac7d8b (ddl,mdl: fix the issue that the table_ids overlap regex is not correct (#52879)):pkg/infoschema/test/clustertablestest/cluster_tables_test.go
Loading