Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl: Support the operation of dropping multi-indexes in one statement #20457

Merged
merged 100 commits into from
Sep 2, 2021
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
100 commits
Select commit Hold shift + click to select a range
f99363c
Merge pull request #1 from pingcap/master
ou-bing Sep 4, 2020
35c9f20
Merge pull request #2 from pingcap/master
ou-bing Sep 21, 2020
b6accd8
Merge pull request #3 from pingcap/master
ou-bing Sep 28, 2020
207e1b5
Merge pull request #4 from pingcap/master
ou-bing Oct 14, 2020
2f12751
add func:DropIndexes
ou-bing Oct 14, 2020
59af45c
add func: onDropIndexes
ou-bing Oct 14, 2020
05e91bb
call func: DropIndexes
ou-bing Oct 14, 2020
3fcdc88
add func: rollingbackDropIndex
ou-bing Oct 15, 2020
550a5c1
Replace onDropIndex with onDropIndexes
ou-bing Oct 15, 2020
d8b29d1
Replace rollingbackDropIndex with rollingbackDropIndexes
ou-bing Oct 15, 2020
0d21ae3
compatible DropIndex
ou-bing Oct 15, 2020
de673ef
remove func: onDropIndex
ou-bing Oct 15, 2020
6fdddca
remove func: rollingbackDropIndex
ou-bing Oct 16, 2020
acb1497
remove func: checkDropIndex
ou-bing Oct 16, 2020
7489ad4
update func: insertJobIntoDeleteRangeTable
ou-bing Oct 16, 2020
213089e
modify variable type
ou-bing Oct 17, 2020
b0301ce
update func: DropIndexes
ou-bing Oct 18, 2020
2035b28
update func: insertJobIntoDeleteRangeTable
ou-bing Oct 18, 2020
6211065
update func: checkDropIndexes
ou-bing Oct 18, 2020
716fa79
update func: insertJobIntoDeleteRangeTable
ou-bing Oct 21, 2020
68032f4
add test case: TestDropIndexes
ou-bing Oct 21, 2020
ff0d5d7
update func: onDropIndexes
ou-bing Oct 21, 2020
7eb7b13
refactor functions: DropIndex、DropIndexes
ou-bing Oct 22, 2020
28bf039
refactor func: checkDropIndexes
ou-bing Oct 23, 2020
d4cc82e
add conditional branch
ou-bing Oct 25, 2020
ec26868
add updateHiddenColumns
ou-bing Oct 25, 2020
1aef3d5
add test case
ou-bing Oct 26, 2020
7355982
add test case
ou-bing Oct 26, 2020
a72172a
fix drop primary key and index
ou-bing Oct 26, 2020
5cd8b49
add test case
ou-bing Oct 26, 2020
fcd5a23
Optimize the code
ou-bing Oct 30, 2020
9fbd625
Optimize the code
ou-bing Oct 30, 2020
385b557
Optimize the code
ou-bing Oct 30, 2020
a7ad554
Merge pull request #5 from pingcap/master
ou-bing Nov 2, 2020
f2576ff
Merge pull request #6 from pingcap/master
ou-bing Nov 5, 2020
ebcb76e
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Nov 5, 2020
4969290
Merge pull request #7 from pingcap/master
ou-bing Nov 9, 2020
0a53b5d
Merge branch 'master' into feature/dropping-multi-indexes
AilinKid Nov 12, 2020
644f4be
Merge branch 'master' into feature/dropping-multi-indexes
AilinKid Nov 12, 2020
c68d7fb
fix bug for drop indexes
ou-bing Nov 12, 2020
f90605c
Merge branch 'master' into feature/dropping-multi-indexes
AilinKid Nov 13, 2020
e56e489
Update logic for modifying job status
ou-bing Nov 13, 2020
2fa1745
Merge branch 'feature/dropping-multi-indexes' of https://github.com/o…
ou-bing Nov 13, 2020
f0f6a01
Merge branch 'master' into feature/dropping-multi-indexes
AilinKid Nov 20, 2020
3b3f3ac
Merge pull request #8 from pingcap/master
ou-bing Nov 20, 2020
0019e4f
Merge pull request #9 from pingcap/master
ou-bing Nov 24, 2020
d8c49c0
Merge pull request #10 from pingcap/master
ou-bing Dec 3, 2020
94c51dd
Merge pull request #11 from pingcap/master
ou-bing Dec 8, 2020
ef4669f
Merge pull request #12 from pingcap/master
ou-bing Dec 25, 2020
2a09406
Merge pull request #13 from pingcap/master
ou-bing Feb 24, 2021
2685390
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Feb 24, 2021
cca227f
typo
ou-bing Feb 24, 2021
3e75858
Merge pull request #14 from pingcap/master
ou-bing Mar 9, 2021
82a80f6
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Mar 9, 2021
65eb3fb
Resume the process of drop index
ou-bing Mar 9, 2021
9a32310
Resume the process of drop index
ou-bing Mar 10, 2021
a9cef9e
typo
ou-bing Mar 10, 2021
db891e1
Merge pull request #15 from pingcap/master
ou-bing Mar 16, 2021
3b83e20
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Mar 16, 2021
75fa9a0
The process of modifying drop
ou-bing Mar 19, 2021
aeccb27
Merge pull request #16 from pingcap/master
ou-bing Mar 19, 2021
71e0491
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Mar 19, 2021
25f9b9c
Optimize code
ou-bing Mar 19, 2021
ffcd5ba
Update test cases
ou-bing Mar 20, 2021
5f06638
Optimize code
ou-bing Mar 23, 2021
4ec2f31
typo
ou-bing Mar 23, 2021
7361b2c
typo
ou-bing Mar 23, 2021
0065255
Merge branch 'feature/dropping-multi-indexes' of https://github.com/o…
ou-bing Mar 23, 2021
76006c0
update test case
ou-bing Mar 28, 2021
36088b1
add func: checkNewIndexes
ou-bing Mar 30, 2021
bcac1fd
Remove rollback mechanism
ou-bing Mar 30, 2021
b24c9db
add para: ifExists
ou-bing Apr 5, 2021
d76d926
Update ddl/rollingback.go
ou-bing Apr 8, 2021
424c956
Update ddl/delete_range.go
ou-bing Apr 8, 2021
e7b376f
typo
ou-bing Apr 11, 2021
70a8b0f
remove func: getSchemaAndTable
ou-bing Apr 11, 2021
d7ad94c
Merge pull request #17 from pingcap/master
ou-bing Apr 20, 2021
c7cfcea
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Apr 20, 2021
760bfdc
add case to `IsJobRollbackable`
ou-bing Apr 21, 2021
ec8ee31
add test case
ou-bing Apr 21, 2021
62af0e8
add test case
ou-bing Apr 24, 2021
7483cdc
add test case
ou-bing Apr 27, 2021
581061c
Merge branch 'pingcap:master' into master
ou-bing May 7, 2021
8ce2b24
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing May 7, 2021
69fd5bb
add test case
ou-bing May 10, 2021
cf9c379
Merge branch 'pingcap:master' into master
ou-bing Jun 19, 2021
237ab27
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Jun 19, 2021
6952ac7
Transform the judgment of "if exists"
ou-bing Jun 28, 2021
73f5c9e
An error is thrown when deleting an invalid index.
ou-bing Jul 11, 2021
6b6c1f4
Merge branch 'pingcap:master' into master
ou-bing Aug 23, 2021
904cb3f
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Aug 23, 2021
ee8c776
update code
ou-bing Aug 23, 2021
27f5db4
update code
ou-bing Aug 25, 2021
6556377
update code
ou-bing Aug 25, 2021
6e46603
Merge branch 'master' into feature/dropping-multi-indexes
zimulala Aug 26, 2021
681cd4a
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Aug 26, 2021
c3107e8
update func: isSameTypeMultiSpecs
ou-bing Sep 1, 2021
0c1bcd1
add test case
ou-bing Sep 1, 2021
e2b6933
Merge branch 'master' into feature/dropping-multi-indexes
ou-bing Sep 1, 2021
f96b07b
Merge branch 'master' into feature/dropping-multi-indexes
zimulala Sep 2, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 212 additions & 0 deletions ddl/db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,111 @@ func (s *testDBSuite4) TestCancelDropPrimaryKey(c *C) {
testCancelDropIndex(c, s.store, s.dom.DDL(), idxName, addIdxSQL, dropIdxSQL)
}

func (s *testDBSuite5) TestCancelDropIndexes(c *C) {
indexesName := []string{"idx_c1", "idx_c2"}
addIdxesSQL := "alter table t add index idx_c1 (c1);alter table t add index idx_c2 (c2);"
dropIdxesSQL := "alter table t drop index idx_c1;alter table t drop index idx_c2;"

store := s.store
d := s.dom.DDL()

tk := testkit.NewTestKit(c, store)
tk.MustExec("use test_db")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(c1 int, c2 int)")
defer tk.MustExec("drop table t;")
for i := 0; i < 5; i++ {
tk.MustExec("insert into t values (?, ?)", i, i)
}
testCases := []struct {
needAddIndex bool
jobState model.JobState
JobSchemaState model.SchemaState
cancelSucc bool
}{
// model.JobStateNone means the jobs is canceled before the first run.
// if we cancel successfully, we need to set needAddIndex to false in the next test case. Otherwise, set needAddIndex to true.
{true, model.JobStateNone, model.StateNone, true},
{false, model.JobStateRunning, model.StateWriteOnly, false},
{true, model.JobStateRunning, model.StateDeleteOnly, false},
{true, model.JobStateRunning, model.StateDeleteReorganization, false},
}
var checkErr error
hook := &ddl.TestDDLCallback{}
var jobID int64
testCase := &testCases[0]
hook.OnJobRunBeforeExported = func(job *model.Job) {
if (job.Type == model.ActionDropIndex || job.Type == model.ActionDropPrimaryKey) &&
job.State == testCase.jobState && job.SchemaState == testCase.JobSchemaState {
jobID = job.ID
jobIDs := []int64{job.ID}
hookCtx := mock.NewContext()
hookCtx.Store = store
err := hookCtx.NewTxn(context.TODO())
if err != nil {
checkErr = errors.Trace(err)
return
}
txn, err := hookCtx.Txn(true)
if err != nil {
checkErr = errors.Trace(err)
return
}

errs, err := admin.CancelJobs(txn, jobIDs)
if err != nil {
checkErr = errors.Trace(err)
return
}
if errs[0] != nil {
checkErr = errors.Trace(errs[0])
return
}
checkErr = txn.Commit(context.Background())
}
}
originalHook := d.GetHook()
d.(ddl.DDLForTest).SetHook(hook)
ctx := tk.Se.(sessionctx.Context)
for i := range testCases {
testCase = &testCases[i]
if testCase.needAddIndex {
tk.MustExec(addIdxesSQL)
}
rs, err := tk.Exec(dropIdxesSQL)
if rs != nil {
rs.Close()
}
t := testGetTableByName(c, ctx, "test_db", "t")

var indexesInfo []*model.IndexInfo
for _, idxName := range indexesName {
indexInfo := t.Meta().FindIndexByName(idxName)
if indexInfo != nil {
indexesInfo = append(indexesInfo, indexInfo)
}
}

if testCase.cancelSucc {
c.Assert(checkErr, IsNil)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:8214]Cancelled DDL job")
c.Assert(indexesInfo, NotNil)
c.Assert(indexesInfo[0].State, Equals, model.StatePublic)
} else {
err1 := admin.ErrCannotCancelDDLJob.GenWithStackByArgs(jobID)
c.Assert(err, IsNil)
c.Assert(checkErr, NotNil)
c.Assert(checkErr.Error(), Equals, err1.Error())
c.Assert(indexesInfo, IsNil)
}
}
d.(ddl.DDLForTest).SetHook(originalHook)
tk.MustExec(addIdxesSQL)
tk.MustExec(dropIdxesSQL)

ou-bing marked this conversation as resolved.
Show resolved Hide resolved
}

// TestCancelDropIndex tests cancel ddl job which type is drop index.
func (s *testDBSuite5) TestCancelDropIndex(c *C) {
idxName := "idx_c2"
Expand Down Expand Up @@ -1481,6 +1586,109 @@ LOOP:
tk.MustExec("drop table test_drop_index")
}

func (s *testDBSuite2) TestDropDuplicateIndexes(c *C) {
tk := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("use test_db")
tk.MustExec("drop table if exists test_drop_duplicate_indexes;")
tk.MustExec("create table test_drop_duplicate_indexes (id int, c1 int, c2 int, primary key(id), key i1(c1), key i2(c2));")

if _, err := tk.Exec("alter table test_drop_duplicate_indexes drop index i1, drop index i1;"); true {
c.Assert(err.Error(), Equals, "[ddl:1091]index i1 doesn't exist")
}

tk.MustExec("alter table test_drop_duplicate_indexes drop index i1, drop index if exists i1;")
tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Note|1091|index i1 doesn't exist"))

if _, err := tk.Exec("alter table test_drop_duplicate_indexes drop index if exists i2, drop index i2;"); true {
c.Assert(err.Error(), Equals, "[ddl:1091]index i2 doesn't exist")
}

tk.MustExec("drop table if exists test_drop_duplicate_indexes")
}

func (s *testDBSuite2) TestDropIndexes(c *C) {
// drop multiple indexes
createSQL := "create table test_drop_indexes (id int, c1 int, c2 int, primary key(id), key i1(c1), key i2(c2))"
dropIdxSQL := "alter table test_drop_indexes drop index i1, drop index i2;"
idxNames := []string{"i1", "i2"}
testDropIndexes(c, s.store, s.lease, createSQL, dropIdxSQL, idxNames)
// test drop primary key and index
dropIdxSQL = "alter table test_drop_indexes drop primary key, drop index i1;"
idxNames = []string{"primary", "i1"}
testDropIndexes(c, s.store, s.lease, createSQL, dropIdxSQL, idxNames)
}

func testDropIndexes(c *C, store kv.Storage, lease time.Duration, createSQL, dropIdxSQL string, idxNames []string) {
tk := testkit.NewTestKit(c, store)
tk.MustExec("use test_db")
tk.MustExec("drop table if exists test_drop_indexes")
tk.MustExec(createSQL)
done := make(chan error, 1)
tk.MustExec("delete from test_drop_indexes")

num := 100
// add some rows
for i := 0; i < num; i++ {
tk.MustExec("insert into test_drop_indexes values (?, ?, ?)", i, i, i)
}
ctx := tk.Se.(sessionctx.Context)
t := testGetTableByName(c, ctx, "test_db", "test_drop_indexes")
var idxs []table.Index
for _, tidx := range t.Indices() {
for _, idxName := range idxNames {
if tidx.Meta().Name.L == idxName {
idxs = append(idxs, tidx)
break
}
}
}
c.Assert(idxs, NotNil)

testddlutil.SessionExecInGoroutine(c, store, dropIdxSQL, done)

ticker := time.NewTicker(lease / 2)
defer ticker.Stop()
LOOP:
for {
select {
case err := <-done:
if err == nil {
break LOOP
}
c.Assert(err, IsNil, Commentf("err:%v", errors.ErrorStack(err)))
case <-ticker.C:
step := 5
// delete some rows, and add some data
for i := num; i < num+step; i++ {
n := rand.Intn(num)
tk.MustExec("update test_drop_indexes set c2 = 1 where c1 = ?", n)
tk.MustExec("insert into test_drop_indexes values (?, ?, ?)", i, i, i)
}
num += step
}
}

// Check in index, it must be no index in KV.
// Make sure there is no index with name c2_index、c3_index.
t = testGetTableByName(c, ctx, "test_db", "test_drop_indexes")
var nidxs []table.Index
for _, tidx := range t.Indices() {
for _, ids := range idxs {
if tidx.Meta().Name.L == ids.Meta().Name.L {
nidxs = append(nidxs, tidx)
}
}
}
c.Assert(nidxs, IsNil)

for _, idx := range idxs {
idx := tables.NewIndex(t.Meta().ID, t.Meta(), idx.Meta())
checkDelRangeDone(c, ctx, idx)
}

tk.MustExec("drop table test_drop_indexes")
}

// TestCancelDropColumn tests cancel ddl job which type is drop column.
func (s *testDBSuite3) TestCancelDropColumn(c *C) {
tk := testkit.NewTestKit(c, s.store)
Expand Down Expand Up @@ -4444,6 +4652,10 @@ func (s *testDBSuite4) TestIfExists(c *C) {
c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1))
tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Note|1091|index idx_c doesn't exist"))

// DROP INDEXES
s.mustExec(tk, c, "alter table t1 drop index if exists idxes_a,drop index if exists idxes_b")
c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(2))

// DROP PARTITION
s.mustExec(tk, c, "drop table if exists t2")
s.mustExec(tk, c, "create table t2 (a int key) partition by range(a) (partition p0 values less than (10), partition p1 values less than (20))")
Expand Down
Loading