From aac260b295c5d4deea9bf7ca917d476614faf9d6 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 11 Jun 2019 14:54:26 +0800 Subject: [PATCH] *: split index region with lower upper syntax (#10409) --- executor/builder.go | 9 +- executor/executor.go | 2 +- executor/executor_test.go | 31 ++++ executor/split.go | 146 +++++++++++++++-- executor/split_test.go | 298 +++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- planner/core/common_plans.go | 9 +- planner/core/planbuilder.go | 118 +++++++++++--- 9 files changed, 571 insertions(+), 48 deletions(-) create mode 100644 executor/split_test.go diff --git a/executor/builder.go b/executor/builder.go index 2b5de81ee15e7..a55bad1f91170 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -194,7 +194,7 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildWindow(v) case *plannercore.SQLBindPlan: return b.buildSQLBindExec(v) - case *plannercore.SplitIndexRegion: + case *plannercore.SplitRegion: return b.buildSplitIndexRegion(v) default: if mp, ok := p.(MockPhysicalPlan); ok { @@ -1261,13 +1261,16 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo return e } -func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitIndexRegion) Executor { +func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitRegion) Executor { base := newBaseExecutor(b.ctx, nil, v.ExplainID()) base.initCap = chunk.ZeroCapacity return &SplitIndexRegionExec{ baseExecutor: base, - table: v.Table, + tableInfo: v.TableInfo, indexInfo: v.IndexInfo, + lower: v.Lower, + upper: v.Upper, + num: v.Num, valueLists: v.ValueLists, } } diff --git a/executor/executor.go b/executor/executor.go index 2b5cbc8acc83c..43017a0e25553 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1386,7 +1386,7 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.InShowWarning = true sc.SetWarnings(vars.StmtCtx.GetWarnings()) } - case *ast.SplitIndexRegionStmt: + case *ast.SplitRegionStmt: sc.IgnoreTruncate = false sc.IgnoreZeroInDate = true sc.AllowInvalidDate = vars.SQLMode.HasAllowInvalidDatesMode() diff --git a/executor/executor_test.go b/executor/executor_test.go index 40bcd461c2705..f7c786e436ec7 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3797,6 +3797,37 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { c.Assert(err, NotNil) terr := errors.Cause(err).(*terror.Error) c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated)) + + // Check min value is more than max value. + tk.MustExec(`split table t index idx1 between (0) and (1000000000) regions 10`) + _, err = tk.Exec(`split table t index idx1 between (2,'a') and (1,'c') regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region `idx1` lower value (2,a) should less than the upper value (1,c)") + + // Check min value is invalid. + _, err = tk.Exec(`split table t index idx1 between () and (1) regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region `idx1` lower value count should more than 0") + + // Check max value is invalid. + _, err = tk.Exec(`split table t index idx1 between (1) and () regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region `idx1` upper value count should more than 0") + + // Check pre-split region num is too large. + _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 10000`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region num is exceed the limit 1000") + + // Check pre-split region num 0 is invalid. + _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 0`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region num should more than 0") + + // Test truncate error msg. + _, err = tk.Exec(`split table t index idx1 between ("aa") and (1000000000) regions 0`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for index column 'b'") } func (s *testSuite) TestIssue10435(c *C) { diff --git a/executor/split.go b/executor/split.go index dc1b75e8018f3..255caaf94b118 100644 --- a/executor/split.go +++ b/executor/split.go @@ -14,13 +14,17 @@ package executor import ( + "bytes" "context" + "encoding/binary" "math" + "github.com/cznic/mathutil" + "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" @@ -31,8 +35,11 @@ import ( type SplitIndexRegionExec struct { baseExecutor - table table.Table + tableInfo *model.TableInfo indexInfo *model.IndexInfo + lower []types.Datum + upper []types.Datum + num int valueLists [][]types.Datum } @@ -48,18 +55,16 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e if !ok { return nil } - regionIDs := make([]uint64, 0, len(e.valueLists)) - index := tables.NewIndex(e.table.Meta().ID, e.table.Meta(), e.indexInfo) - for _, values := range e.valueLists { - idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, math.MinInt64, nil) - if err != nil { - return err - } - + splitIdxKeys, err := e.getSplitIdxKeys() + if err != nil { + return err + } + regionIDs := make([]uint64, 0, len(splitIdxKeys)) + for _, idxKey := range splitIdxKeys { regionID, err := s.SplitRegionAndScatter(idxKey) if err != nil { logutil.Logger(context.Background()).Warn("split table index region failed", - zap.String("table", e.table.Meta().Name.L), + zap.String("table", e.tableInfo.Name.L), zap.String("index", e.indexInfo.Name.L), zap.Error(err)) continue @@ -75,10 +80,127 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e if err != nil { logutil.Logger(context.Background()).Warn("wait scatter region failed", zap.Uint64("regionID", regionID), - zap.String("table", e.table.Meta().Name.L), + zap.String("table", e.tableInfo.Name.L), zap.String("index", e.indexInfo.Name.L), zap.Error(err)) } } return nil } + +func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { + var idxKeys [][]byte + if e.num > 0 { + idxKeys = make([][]byte, 0, e.num) + } else { + idxKeys = make([][]byte, 0, len(e.valueLists)+1) + } + // Split in the start of the index key. + startIdxKey := tablecodec.EncodeTableIndexPrefix(e.tableInfo.ID, e.indexInfo.ID) + idxKeys = append(idxKeys, startIdxKey) + + index := tables.NewIndex(e.tableInfo.ID, e.tableInfo, e.indexInfo) + // Split index regions by user specified value lists. + if len(e.valueLists) > 0 { + for _, v := range e.valueLists { + idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, v, math.MinInt64, nil) + if err != nil { + return nil, err + } + idxKeys = append(idxKeys, idxKey) + } + return idxKeys, nil + } + // Split index regions by lower, upper value and calculate the step by (upper - lower)/num. + lowerIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.lower, math.MinInt64, nil) + if err != nil { + return nil, err + } + // Use math.MinInt64 as handle_id for the upper index key to avoid affecting calculate split point. + // If use math.MaxInt64 here, test of `TestSplitIndex` will report error. + upperIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.upper, math.MinInt64, nil) + if err != nil { + return nil, err + } + if bytes.Compare(lowerIdxKey, upperIdxKey) >= 0 { + lowerStr, err1 := datumSliceToString(e.lower) + upperStr, err2 := datumSliceToString(e.upper) + if err1 != nil || err2 != nil { + return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper) + } + return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, lowerStr, upperStr) + } + return getValuesList(lowerIdxKey, upperIdxKey, e.num, idxKeys), nil +} + +// getValuesList is used to get `num` values between lower and upper value. +// To Simplify the explain, suppose lower and upper value type is int64, and lower=0, upper=100, num=10, +// then calculate the step=(upper-lower)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. +// Then the function will return [10,20,30,40,50,60,70,80,90]. +// The difference is the value type of upper,lower is []byte, So I use getUint64FromBytes to convert []byte to uint64. +func getValuesList(lower, upper []byte, num int, valuesList [][]byte) [][]byte { + commonPrefixIdx := longestCommonPrefixLen(lower, upper) + step := getStepValue(lower[commonPrefixIdx:], upper[commonPrefixIdx:], num) + startV := getUint64FromBytes(lower[commonPrefixIdx:], 0) + // To get `num` regions, only need to split `num-1` idx keys. + buf := make([]byte, 8) + for i := 0; i < num-1; i++ { + value := make([]byte, 0, commonPrefixIdx+8) + value = append(value, lower[:commonPrefixIdx]...) + startV += step + binary.BigEndian.PutUint64(buf, startV) + value = append(value, buf...) + valuesList = append(valuesList, value) + } + return valuesList +} + +// longestCommonPrefixLen gets the longest common prefix byte length. +func longestCommonPrefixLen(s1, s2 []byte) int { + l := mathutil.Min(len(s1), len(s2)) + i := 0 + for ; i < l; i++ { + if s1[i] != s2[i] { + break + } + } + return i +} + +// getStepValue gets the step of between the lower and upper value. step = (upper-lower)/num. +// Convert byte slice to uint64 first. +func getStepValue(lower, upper []byte, num int) uint64 { + lowerUint := getUint64FromBytes(lower, 0) + upperUint := getUint64FromBytes(upper, 0xff) + return (upperUint - lowerUint) / uint64(num) +} + +// getUint64FromBytes gets a uint64 from the `bs` byte slice. +// If len(bs) < 8, then padding with `pad`. +func getUint64FromBytes(bs []byte, pad byte) uint64 { + buf := bs + if len(buf) < 8 { + buf = make([]byte, 0, 8) + buf = append(buf, bs...) + for i := len(buf); i < 8; i++ { + buf = append(buf, pad) + } + } + return binary.BigEndian.Uint64(buf) +} + +func datumSliceToString(ds []types.Datum) (string, error) { + str := "(" + for i, d := range ds { + s, err := d.ToString() + if err != nil { + return str, err + } + if i > 0 { + str += "," + } + str += s + } + str += ")" + return str, nil +} diff --git a/executor/split_test.go b/executor/split_test.go new file mode 100644 index 0000000000000..c73d37a5d65dd --- /dev/null +++ b/executor/split_test.go @@ -0,0 +1,298 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "bytes" + "encoding/binary" + "math" + "math/rand" + + . "github.com/pingcap/check" + "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/mock" +) + +var _ = Suite(&testSplitIndex{}) + +type testSplitIndex struct { +} + +func (s *testSplitIndex) SetUpSuite(c *C) { +} + +func (s *testSplitIndex) TearDownSuite(c *C) { +} + +func (s *testSplitIndex) TestLongestCommonPrefixLen(c *C) { + cases := []struct { + s1 string + s2 string + l int + }{ + {"", "", 0}, + {"", "a", 0}, + {"a", "", 0}, + {"a", "a", 1}, + {"ab", "a", 1}, + {"a", "ab", 1}, + {"b", "ab", 0}, + {"ba", "ab", 0}, + } + + for _, ca := range cases { + re := longestCommonPrefixLen([]byte(ca.s1), []byte(ca.s2)) + c.Assert(re, Equals, ca.l) + } +} + +func (s *testSplitIndex) TestgetStepValue(c *C) { + cases := []struct { + lower []byte + upper []byte + l int + v uint64 + }{ + {[]byte{}, []byte{}, 0, math.MaxUint64}, + {[]byte{0}, []byte{128}, 0, binary.BigEndian.Uint64([]byte{128, 255, 255, 255, 255, 255, 255, 255})}, + {[]byte{'a'}, []byte{'z'}, 0, binary.BigEndian.Uint64([]byte{'z' - 'a', 255, 255, 255, 255, 255, 255, 255})}, + {[]byte("abc"), []byte{'z'}, 0, binary.BigEndian.Uint64([]byte{'z' - 'a', 255 - 'b', 255 - 'c', 255, 255, 255, 255, 255})}, + {[]byte("abc"), []byte("xyz"), 0, binary.BigEndian.Uint64([]byte{'x' - 'a', 'y' - 'b', 'z' - 'c', 255, 255, 255, 255, 255})}, + {[]byte("abc"), []byte("axyz"), 1, binary.BigEndian.Uint64([]byte{'x' - 'b', 'y' - 'c', 'z', 255, 255, 255, 255, 255})}, + {[]byte("abc0123456"), []byte("xyz01234"), 0, binary.BigEndian.Uint64([]byte{'x' - 'a', 'y' - 'b', 'z' - 'c', 0, 0, 0, 0, 0})}, + } + + for _, ca := range cases { + l := longestCommonPrefixLen(ca.lower, ca.upper) + c.Assert(l, Equals, ca.l) + v0 := getStepValue(ca.lower[l:], ca.upper[l:], 1) + c.Assert(v0, Equals, ca.v) + } +} + +func (s *testSplitIndex) TestSplitIndex(c *C) { + tbInfo := &model.TableInfo{ + Name: model.NewCIStr("t1"), + ID: rand.Int63(), + Columns: []*model.ColumnInfo{ + { + Name: model.NewCIStr("c0"), + ID: 1, + Offset: 1, + DefaultValue: 0, + State: model.StatePublic, + FieldType: *types.NewFieldType(mysql.TypeLong), + }, + }, + } + idxCols := []*model.IndexColumn{{Name: tbInfo.Columns[0].Name, Offset: 0, Length: types.UnspecifiedLength}} + idxInfo := &model.IndexInfo{ + ID: 1, + Name: model.NewCIStr("idx1"), + Table: model.NewCIStr("t1"), + Columns: idxCols, + State: model.StatePublic, + } + + // Test for int index. + // range is 0 ~ 100, and split into 10 region. + // So 10 regions range is like below, left close right open interval: + // region1: [-inf ~ 10) + // region2: [10 ~ 20) + // region3: [20 ~ 30) + // region4: [30 ~ 40) + // region5: [40 ~ 50) + // region6: [50 ~ 60) + // region7: [60 ~ 70) + // region8: [70 ~ 80) + // region9: [80 ~ 90) + // region10: [90 ~ +inf) + ctx := mock.NewContext() + e := &SplitIndexRegionExec{ + baseExecutor: newBaseExecutor(ctx, nil, nil), + tableInfo: tbInfo, + indexInfo: idxInfo, + lower: []types.Datum{types.NewDatum(0)}, + upper: []types.Datum{types.NewDatum(100)}, + num: 10, + } + valueList, err := e.getSplitIdxKeys() + c.Assert(err, IsNil) + c.Assert(len(valueList), Equals, e.num) + + cases := []struct { + value int + lessEqualIdx int + }{ + {-1, 0}, + {0, 0}, + {1, 0}, + {10, 1}, + {11, 1}, + {20, 2}, + {21, 2}, + {31, 3}, + {41, 4}, + {51, 5}, + {61, 6}, + {71, 7}, + {81, 8}, + {91, 9}, + {100, 9}, + {1000, 9}, + } + + index := tables.NewIndex(tbInfo.ID, tbInfo, idxInfo) + for _, ca := range cases { + // test for minInt64 handle + idxValue, _, err := index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MinInt64, nil) + c.Assert(err, IsNil) + idx := searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + + // Test for max int64 handle. + idxValue, _, err = index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MaxInt64, nil) + c.Assert(err, IsNil) + idx = searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + } + // Test for varchar index. + // range is a ~ z, and split into 26 region. + // So 26 regions range is like below: + // region1: [-inf ~ b) + // region2: [b ~ c) + // . + // . + // . + // region26: [y ~ +inf) + e.lower = []types.Datum{types.NewDatum("a")} + e.upper = []types.Datum{types.NewDatum("z")} + e.num = 26 + // change index column type to varchar + tbInfo.Columns[0].FieldType = *types.NewFieldType(mysql.TypeVarchar) + + valueList, err = e.getSplitIdxKeys() + c.Assert(err, IsNil) + c.Assert(len(valueList), Equals, e.num) + + cases2 := []struct { + value string + lessEqualIdx int + }{ + {"", 0}, + {"a", 0}, + {"abcde", 0}, + {"b", 1}, + {"bzzzz", 1}, + {"c", 2}, + {"czzzz", 2}, + {"z", 25}, + {"zabcd", 25}, + } + + for _, ca := range cases2 { + // test for minInt64 handle + idxValue, _, err := index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MinInt64, nil) + c.Assert(err, IsNil) + idx := searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + + // Test for max int64 handle. + idxValue, _, err = index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MaxInt64, nil) + c.Assert(err, IsNil) + idx = searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + } + + // Test for timestamp index. + // range is 2010-01-01 00:00:00 ~ 2020-01-01 00:00:00, and split into 10 region. + // So 10 regions range is like below: + // region1: [-inf ~ 2011-01-01 00:00:00) + // region2: [2011-01-01 00:00:00 ~ 2012-01-01 00:00:00) + // . + // . + // . + // region10: [2019-01-01 00:00:00 ~ +inf) + lowerTime := types.Time{ + Time: types.FromDate(2010, 1, 1, 0, 0, 0, 0), + Type: mysql.TypeTimestamp, + } + upperTime := types.Time{ + Time: types.FromDate(2020, 1, 1, 0, 0, 0, 0), + Type: mysql.TypeTimestamp, + } + e.lower = []types.Datum{types.NewDatum(lowerTime)} + e.upper = []types.Datum{types.NewDatum(upperTime)} + e.num = 10 + + // change index column type to timestamp + tbInfo.Columns[0].FieldType = *types.NewFieldType(mysql.TypeTimestamp) + + valueList, err = e.getSplitIdxKeys() + c.Assert(err, IsNil) + c.Assert(len(valueList), Equals, e.num) + + cases3 := []struct { + value types.MysqlTime + lessEqualIdx int + }{ + {types.FromDate(2009, 11, 20, 12, 50, 59, 0), 0}, + {types.FromDate(2010, 1, 1, 0, 0, 0, 0), 0}, + {types.FromDate(2011, 12, 31, 23, 59, 59, 0), 1}, + {types.FromDate(2011, 2, 1, 0, 0, 0, 0), 1}, + {types.FromDate(2012, 3, 1, 0, 0, 0, 0), 2}, + {types.FromDate(2013, 4, 1, 0, 0, 0, 0), 3}, + {types.FromDate(2014, 5, 1, 0, 0, 0, 0), 4}, + {types.FromDate(2015, 6, 1, 0, 0, 0, 0), 5}, + {types.FromDate(2016, 8, 1, 0, 0, 0, 0), 6}, + {types.FromDate(2017, 9, 1, 0, 0, 0, 0), 7}, + {types.FromDate(2018, 10, 1, 0, 0, 0, 0), 8}, + {types.FromDate(2019, 11, 1, 0, 0, 0, 0), 9}, + {types.FromDate(2020, 12, 1, 0, 0, 0, 0), 9}, + {types.FromDate(2030, 12, 1, 0, 0, 0, 0), 9}, + } + + for _, ca := range cases3 { + value := types.Time{ + Time: ca.value, + Type: mysql.TypeTimestamp, + } + // test for min int64 handle + idxValue, _, err := index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(value)}, math.MinInt64, nil) + c.Assert(err, IsNil) + idx := searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + + // Test for max int64 handle. + idxValue, _, err = index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(value)}, math.MaxInt64, nil) + c.Assert(err, IsNil) + idx = searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + } +} + +func searchLessEqualIdx(valueList [][]byte, value []byte) int { + idx := -1 + for i, v := range valueList { + if bytes.Compare(value, v) >= 0 { + idx = i + continue + } + break + } + return idx +} diff --git a/go.mod b/go.mod index b2e70e876ddbc..3a29cf4b42b3d 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190528074401-b942b3f4108f github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190604062806-1e93bd08fe82 + github.com/pingcap/parser v0.0.0-20190605085902-82be48fe0267 github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 2afd2e88fabea..380316f3e39cb 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190528074401-b942b3f4108f/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190604062806-1e93bd08fe82 h1:Xir4G43GOHOFioCI+wtVTv/ssCkGnMzZHzpF7ZfDxEs= -github.com/pingcap/parser v0.0.0-20190604062806-1e93bd08fe82/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190605085902-82be48fe0267 h1:LAR69Ocf7MIxHuh0uxr4JjRfPiFpKHq6aqEkvD63Z/k= +github.com/pingcap/parser v0.0.0-20190605085902-82be48fe0267/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index bbb9b3497c270..3c3d8e034e058 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -479,12 +479,15 @@ type LoadStats struct { Path string } -// SplitIndexRegion represents a split index regions plan. -type SplitIndexRegion struct { +// SplitRegion represents a split regions plan. +type SplitRegion struct { baseSchemaProducer - Table table.Table + TableInfo *model.TableInfo IndexInfo *model.IndexInfo + Lower []types.Datum + Upper []types.Datum + Num int ValueLists [][]types.Datum } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index a87abfb89642d..117aa12a80c01 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" + "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/ranger" ) @@ -279,8 +280,8 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) { return b.buildDropBindPlan(x) case *ast.ChangeStmt: return b.buildChange(x) - case *ast.SplitIndexRegionStmt: - return b.buildSplitIndexRegion(x) + case *ast.SplitRegionStmt: + return b.buildSplitRegion(x) } return nil, ErrUnsupportedType.GenWithStack("Unsupported type %T", node) } @@ -1619,44 +1620,109 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { return p } -func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Plan, error) { +const maxSplitRegionNum = 1000 + +func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) { tblInfo := node.Table.TableInfo - indexInfo := tblInfo.FindIndexByName(strings.ToLower(node.IndexName)) + indexInfo := tblInfo.FindIndexByName(node.IndexName.L) if indexInfo == nil { return nil, ErrKeyDoesNotExist.GenWithStackByArgs(node.IndexName, tblInfo.Name) } + mockTablePlan := LogicalTableDual{}.Init(b.ctx) + schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo) + mockTablePlan.SetSchema(schema) - indexValues := make([][]types.Datum, 0, len(node.ValueLists)) - for i, valuesItem := range node.ValueLists { - if len(valuesItem) > len(indexInfo.Columns) { - return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) - } - valueList := make([]types.Datum, 0, len(valuesItem)) - for j, valueItem := range valuesItem { - x, ok := valueItem.(*driver.ValueExpr) - if !ok { - return nil, errors.New("expect constant values") + p := &SplitRegion{ + TableInfo: tblInfo, + IndexInfo: indexInfo, + } + // Split index regions by user specified value lists. + if len(node.SplitOpt.ValueLists) > 0 { + indexValues := make([][]types.Datum, 0, len(node.SplitOpt.ValueLists)) + for i, valuesItem := range node.SplitOpt.ValueLists { + if len(valuesItem) > len(indexInfo.Columns) { + return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) } - colOffset := indexInfo.Columns[j].Offset - value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) + values, err := b.convertValue2ColumnType(valuesItem, mockTablePlan, indexInfo, tblInfo) if err != nil { return nil, err } + indexValues = append(indexValues, values) + } + p.ValueLists = indexValues + return p, nil + } - valueList = append(valueList, value) + // Split index regions by lower, upper value. + checkLowerUpperValue := func(valuesItem []ast.ExprNode, name string) ([]types.Datum, error) { + if len(valuesItem) == 0 { + return nil, errors.Errorf("Split index region `%v` %s value count should more than 0", indexInfo.Name, name) } - indexValues = append(indexValues, valueList) + if len(valuesItem) > len(indexInfo.Columns) { + return nil, errors.Errorf("Split index region `%v` Column count doesn't match value count at %v", indexInfo.Name, name) + } + return b.convertValue2ColumnType(valuesItem, mockTablePlan, indexInfo, tblInfo) } - tableInPlan, ok := b.is.TableByID(tblInfo.ID) - if !ok { - return nil, errors.Errorf("Can't get table %s.", tblInfo.Name.O) + lowerValues, err := checkLowerUpperValue(node.SplitOpt.Lower, "lower") + if err != nil { + return nil, err + } + upperValues, err := checkLowerUpperValue(node.SplitOpt.Upper, "upper") + if err != nil { + return nil, err + } + p.Lower = lowerValues + p.Upper = upperValues + + if node.SplitOpt.Num > maxSplitRegionNum { + return nil, errors.Errorf("Split index region num is exceed the limit %v", maxSplitRegionNum) + } else if node.SplitOpt.Num < 1 { + return nil, errors.Errorf("Split index region num should more than 0") } - return &SplitIndexRegion{ - Table: tableInPlan, - IndexInfo: indexInfo, - ValueLists: indexValues, - }, nil + p.Num = int(node.SplitOpt.Num) + return p, nil +} +func (b *PlanBuilder) convertValue2ColumnType(valuesItem []ast.ExprNode, mockTablePlan LogicalPlan, indexInfo *model.IndexInfo, tblInfo *model.TableInfo) ([]types.Datum, error) { + values := make([]types.Datum, 0, len(valuesItem)) + for j, valueItem := range valuesItem { + var expr expression.Expression + var err error + switch x := valueItem.(type) { + case *driver.ValueExpr: + expr = &expression.Constant{ + Value: x.Datum, + RetType: &x.Type, + } + default: + expr, _, err = b.rewrite(valueItem, mockTablePlan, nil, true) + if err != nil { + return nil, err + } + } + constant, ok := expr.(*expression.Constant) + if !ok { + return nil, errors.New("expect constant values") + } + value, err := constant.Eval(chunk.Row{}) + if err != nil { + return nil, err + } + colOffset := indexInfo.Columns[j].Offset + value1, err := value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) + if err != nil { + if !types.ErrTruncated.Equal(err) { + return nil, err + } + valStr, err1 := value.ToString() + if err1 != nil { + return nil, err + } + return nil, types.ErrTruncated.GenWithStack("Incorrect value: '%-.128s' for index column '%.192s'", valStr, tblInfo.Columns[colOffset].Name.O) + } + values = append(values, value1) + } + return values, nil } func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) {