From e67ecf458184dbaa250205dcf7e689654a5493ff Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 9 May 2019 19:12:12 +0800 Subject: [PATCH 01/21] *: split index region with min max num syntax --- executor/builder.go | 5 +- executor/executor_test.go | 21 +++ executor/split.go | 134 ++++++++++++++-- executor/split_test.go | 300 +++++++++++++++++++++++++++++++++++ go.mod | 2 + go.sum | 2 + planner/core/common_plans.go | 5 +- planner/core/planbuilder.go | 92 ++++++++--- 8 files changed, 528 insertions(+), 33 deletions(-) create mode 100644 executor/split_test.go diff --git a/executor/builder.go b/executor/builder.go index 915ed68ebccc7..627dae98c31cf 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1254,8 +1254,11 @@ func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitIndexRegion) base.initCap = chunk.ZeroCapacity return &SplitIndexRegionExec{ baseExecutor: base, - table: v.Table, + tableInfo: v.TableInfo, indexInfo: v.IndexInfo, + min: v.Min, + max: v.Max, + num: v.Num, valueLists: v.ValueLists, } } diff --git a/executor/executor_test.go b/executor/executor_test.go index c21b8104fae28..3ad795bab6a70 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3755,6 +3755,27 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { c.Assert(err, NotNil) terr := errors.Cause(err).(*terror.Error) c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated)) + + // Check min value is more than max value. + tk.MustExec(`split table t index idx1 min (0) max (1000000000) num 10`) + _, err = tk.Exec(`split table t index idx1 min (2) max (1) num 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region `idx1` min value [{1 0 0 0 2 [] }] should less than the max value [{1 0 0 0 1 [] }]") + + // Check min value is invalid. + _, err = tk.Exec(`split table t index idx1 min () max (1) num 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region `idx1` min value count should more than 0") + + // Check max value is invalid. + _, err = tk.Exec(`split table t index idx1 min (1) max () num 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region `idx1` max value count should more than 0") + + // Check pre-split region num is too large. + _, err = tk.Exec(`split table t index idx1 min (0) max (1000000000) num 10000`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region num is exceed the limit 1000") } type testOOMSuite struct { diff --git a/executor/split.go b/executor/split.go index dc1b75e8018f3..d3c316079b04b 100644 --- a/executor/split.go +++ b/executor/split.go @@ -14,12 +14,14 @@ package executor import ( + "bytes" "context" + "encoding/binary" "math" + "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -31,8 +33,11 @@ import ( type SplitIndexRegionExec struct { baseExecutor - table table.Table + tableInfo *model.TableInfo indexInfo *model.IndexInfo + min []types.Datum + max []types.Datum + num int valueLists [][]types.Datum } @@ -48,18 +53,16 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e if !ok { return nil } - regionIDs := make([]uint64, 0, len(e.valueLists)) - index := tables.NewIndex(e.table.Meta().ID, e.table.Meta(), e.indexInfo) - for _, values := range e.valueLists { - idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, math.MinInt64, nil) - if err != nil { - return err - } - + splitIdxKeys, err := e.getSplitIdxKeys() + if err != nil { + return err + } + regionIDs := make([]uint64, 0, len(splitIdxKeys)) + for _, idxKey := range splitIdxKeys { regionID, err := s.SplitRegionAndScatter(idxKey) if err != nil { logutil.Logger(context.Background()).Warn("split table index region failed", - zap.String("table", e.table.Meta().Name.L), + zap.String("table", e.tableInfo.Name.L), zap.String("index", e.indexInfo.Name.L), zap.Error(err)) continue @@ -75,10 +78,117 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e if err != nil { logutil.Logger(context.Background()).Warn("wait scatter region failed", zap.Uint64("regionID", regionID), - zap.String("table", e.table.Meta().Name.L), + zap.String("table", e.tableInfo.Name.L), zap.String("index", e.indexInfo.Name.L), zap.Error(err)) } } return nil } + +func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { + idxKeys := make([][]byte, 0, len(e.valueLists)) + index := tables.NewIndex(e.tableInfo.ID, e.tableInfo, e.indexInfo) + if len(e.valueLists) > 0 { + for _, v := range e.valueLists { + idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, v, math.MinInt64, nil) + if err != nil { + return nil, err + } + idxKeys = append(idxKeys, idxKey) + } + return idxKeys, nil + } + minIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.min, math.MinInt64, nil) + if err != nil { + return nil, err + } + maxIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.max, math.MinInt64, nil) + if err != nil { + return nil, err + } + if bytes.Compare(minIdxKey, maxIdxKey) >= 0 { + return nil, errors.Errorf("Split index region `%v` min value %v should less than the max value %v", e.indexInfo.Name, e.min, e.max) + } + idxKeys = make([][]byte, 0, e.num+1) + return getValuesList(minIdxKey, maxIdxKey, e.num), nil +} + +func longestCommonPrefixLen(s1, s2 []byte) int { + l := len(s1) + if len(s2) < len(s1) { + l = len(s2) + } + i := 0 + for ; i < l; i++ { + if s1[i] != s2[i] { + break + } + } + return i +} + +func getDiffBytesValue(startIdx int, min, max []byte) uint64 { + l := len(min) + if len(max) < len(min) { + l = len(max) + } + if l-startIdx > 8 { + l = startIdx + 8 + } + diff := make([]byte, 0, 8) + for i := startIdx; i < l; i++ { + diff = append(diff, max[i]-min[i]) + } + if len(max) > l { + for i := l; i < len(max); i++ { + diff = append(diff, max[i]) + if len(diff) >= 8 { + break + } + } + } + if len(min) > l { + for i := l; i < len(min); i++ { + diff = append(diff, 0xff-min[i]) + if len(diff) >= 8 { + break + } + } + } + + for i := len(diff); i < 8; i++ { + diff = append(diff, 0xff) + } + diffValue := binary.BigEndian.Uint64(diff) + return diffValue +} + +func getValuesList(min, max []byte, num int) [][]byte { + startIdx := longestCommonPrefixLen(min, max) + diffValue := getDiffBytesValue(startIdx, min, max) + step := diffValue / uint64(num) + + startValueTemp := min[startIdx:] + if len(startValueTemp) > 8 { + startValueTemp = startValueTemp[:8] + } + startValue := make([]byte, 0, 8) + startValue = append(startValue, startValueTemp...) + for i := len(startValue); i < 8; i++ { + startValue = append(startValue, 0) + } + startV := binary.BigEndian.Uint64(startValue) + valuesList := make([][]byte, 0, num+1) + valuesList = append(valuesList, min) + tmp := make([]byte, 8) + for i := 0; i < num; i++ { + value := make([]byte, 0, startIdx+8) + value = append(value, min[:startIdx]...) + startV += step + binary.BigEndian.PutUint64(tmp, startV) + value = append(value, tmp...) + valuesList = append(valuesList, value) + } + return valuesList +} diff --git a/executor/split_test.go b/executor/split_test.go new file mode 100644 index 0000000000000..50acfc065ad18 --- /dev/null +++ b/executor/split_test.go @@ -0,0 +1,300 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "bytes" + "encoding/binary" + "math" + "math/rand" + + . "github.com/pingcap/check" + "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/mock" +) + +var _ = Suite(&testSplitIndex{}) + +type testSplitIndex struct { +} + +func (s *testSplitIndex) SetUpSuite(c *C) { +} + +func (s *testSplitIndex) TearDownSuite(c *C) { +} + +func (s *testSplitIndex) TestLongestCommonPrefixLen(c *C) { + cases := []struct { + s1 string + s2 string + l int + }{ + {"", "", 0}, + {"", "a", 0}, + {"a", "", 0}, + {"a", "a", 1}, + {"ab", "a", 1}, + {"a", "ab", 1}, + {"b", "ab", 0}, + {"ba", "ab", 0}, + } + + for _, ca := range cases { + re := longestCommonPrefixLen([]byte(ca.s1), []byte(ca.s2)) + c.Assert(re, Equals, ca.l) + } +} + +func (s *testSplitIndex) TestGetDiffBytesValue(c *C) { + cases := []struct { + min []byte + max []byte + l int + v uint64 + }{ + {[]byte{}, []byte{}, 0, math.MaxUint64}, + {[]byte{0}, []byte{128}, 0, binary.BigEndian.Uint64([]byte{128, 255, 255, 255, 255, 255, 255, 255})}, + {[]byte{'a'}, []byte{'z'}, 0, binary.BigEndian.Uint64([]byte{'z' - 'a', 255, 255, 255, 255, 255, 255, 255})}, + {[]byte("abc"), []byte{'z'}, 0, binary.BigEndian.Uint64([]byte{'z' - 'a', 255 - 'b', 255 - 'c', 255, 255, 255, 255, 255})}, + {[]byte("abc"), []byte("xyz"), 0, binary.BigEndian.Uint64([]byte{'x' - 'a', 'y' - 'b', 'z' - 'c', 255, 255, 255, 255, 255})}, + {[]byte("abc"), []byte("axyz"), 1, binary.BigEndian.Uint64([]byte{'x' - 'b', 'y' - 'c', 'z', 255, 255, 255, 255, 255})}, + {[]byte("abc0123456"), []byte("xyz01234"), 0, binary.BigEndian.Uint64([]byte{'x' - 'a', 'y' - 'b', 'z' - 'c', 0, 0, 0, 0, 0})}, + } + + for _, ca := range cases { + l0 := longestCommonPrefixLen(ca.min, ca.max) + c.Assert(l0, Equals, ca.l) + v0 := getDiffBytesValue(l0, ca.min, ca.max) + c.Assert(v0, Equals, ca.v) + } +} + +func (s *testSplitIndex) TestSplitIndex(c *C) { + tbInfo := &model.TableInfo{ + Name: model.NewCIStr("t1"), + ID: rand.Int63(), + Columns: []*model.ColumnInfo{ + { + Name: model.NewCIStr("c0"), + ID: 1, + Offset: 1, + DefaultValue: 0, + State: model.StatePublic, + FieldType: *types.NewFieldType(mysql.TypeLong), + }, + }, + } + idxCols := []*model.IndexColumn{{Name: tbInfo.Columns[0].Name, Offset: 0, Length: types.UnspecifiedLength}} + idxInfo := &model.IndexInfo{ + ID: 1, + Name: model.NewCIStr("idx1"), + Table: model.NewCIStr("t1"), + Columns: idxCols, + State: model.StatePublic, + } + + // Test for int index. + // range is 0 ~ 100, and split into 10 region. + // So 10 regions range is like below: + // region0: -inf ~ 0 + // region1: 0 ~ 10 + // region2: 10 ~ 20 + // region3: 20 ~ 30 + // region4: 30 ~ 40 + // region5: 40 ~ 50 + // region6: 50 ~ 60 + // region7: 60 ~ 70 + // region8: 70 ~ 80 + // region9: 80 ~ 80 + // region10: 90 ~ +inf + ctx := mock.NewContext() + e := &SplitIndexRegionExec{ + baseExecutor: newBaseExecutor(ctx, nil, nil), + tableInfo: tbInfo, + indexInfo: idxInfo, + min: []types.Datum{types.NewDatum(0)}, + max: []types.Datum{types.NewDatum(100)}, + num: 10, + } + valueList, err := e.getSplitIdxKeys() + c.Assert(err, IsNil) + c.Assert(len(valueList), Equals, e.num+1) + + cases := []struct { + value int + lessEqualIdx int + }{ + {-1, -1}, + {0, 0}, + {1, 0}, + {10, 1}, + {11, 1}, + {20, 2}, + {21, 2}, + {21, 2}, + {31, 3}, + {41, 4}, + {51, 5}, + {61, 6}, + {71, 7}, + {81, 8}, + {91, 9}, + {100, 10}, + {1000, 10}, + } + + index := tables.NewIndex(tbInfo.ID, tbInfo, idxInfo) + for _, ca := range cases { + // test for minInt64 handle + idxValue, _, err := index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MinInt64, nil) + c.Assert(err, IsNil) + idx := searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + + // Test for max int64 handle. + idxValue, _, err = index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MaxInt64, nil) + c.Assert(err, IsNil) + idx = searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + } + // Test for varchar index. + // range is a ~ z, and split into 26 region. + // So 26 regions range is like below: + // region0: -inf ~ a + // region1: a ~ b + // . + // . + // . + // region26: z ~ +inf + e.min = []types.Datum{types.NewDatum("a")} + e.max = []types.Datum{types.NewDatum("z")} + e.num = 26 + // change index column type to varchar + tbInfo.Columns[0].FieldType = *types.NewFieldType(mysql.TypeVarchar) + + valueList, err = e.getSplitIdxKeys() + c.Assert(err, IsNil) + c.Assert(len(valueList), Equals, e.num+1) + + cases2 := []struct { + value string + lessEqualIdx int + }{ + {"", -1}, + {"a", 0}, + {"abcde", 0}, + {"b", 1}, + {"bzzzz", 1}, + {"c", 2}, + {"czzzz", 2}, + {"z", 26}, + {"zabcd", 26}, + } + + for _, ca := range cases2 { + // test for minInt64 handle + idxValue, _, err := index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MinInt64, nil) + c.Assert(err, IsNil) + idx := searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + + // Test for max int64 handle. + idxValue, _, err = index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(ca.value)}, math.MaxInt64, nil) + c.Assert(err, IsNil) + idx = searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + } + + // Test for timestamp index. + // range is 2010-01-01 00:00:00 ~ 2020-01-01 00:00:00, and split into 10 region. + // So 10 regions range is like below: + // region0: -inf ~ 2010-01-01 00:00:00 + // region1: 2010-01-01 00:00:00 ~ 2011-01-01 00:00:00 + // . + // . + // . + // region10: 2020-01-01 00:00:00 ~ +inf + minTime := types.Time{ + Time: types.FromDate(2010, 1, 1, 0, 0, 0, 0), + Type: mysql.TypeTimestamp, + } + maxTime := types.Time{ + Time: types.FromDate(2020, 1, 1, 0, 0, 0, 0), + Type: mysql.TypeTimestamp, + } + e.min = []types.Datum{types.NewDatum(minTime)} + e.max = []types.Datum{types.NewDatum(maxTime)} + e.num = 10 + + // change index column type to timestamp + tbInfo.Columns[0].FieldType = *types.NewFieldType(mysql.TypeTimestamp) + + valueList, err = e.getSplitIdxKeys() + c.Assert(err, IsNil) + c.Assert(len(valueList), Equals, e.num+1) + + cases3 := []struct { + value types.MysqlTime + lessEqualIdx int + }{ + {types.FromDate(2009, 11, 20, 12, 50, 59, 0), -1}, + {types.FromDate(2010, 1, 1, 0, 0, 0, 0), 0}, + {types.FromDate(2011, 12, 31, 23, 59, 59, 0), 1}, + {types.FromDate(2011, 2, 1, 0, 0, 0, 0), 1}, + {types.FromDate(2012, 3, 1, 0, 0, 0, 0), 2}, + {types.FromDate(2013, 4, 1, 0, 0, 0, 0), 3}, + {types.FromDate(2014, 5, 1, 0, 0, 0, 0), 4}, + {types.FromDate(2015, 6, 1, 0, 0, 0, 0), 5}, + {types.FromDate(2016, 8, 1, 0, 0, 0, 0), 6}, + {types.FromDate(2017, 9, 1, 0, 0, 0, 0), 7}, + {types.FromDate(2018, 10, 1, 0, 0, 0, 0), 8}, + {types.FromDate(2019, 11, 1, 0, 0, 0, 0), 9}, + {types.FromDate(2020, 12, 1, 0, 0, 0, 0), 10}, + {types.FromDate(2030, 12, 1, 0, 0, 0, 0), 10}, + } + + for _, ca := range cases3 { + value := types.Time{ + Time: ca.value, + Type: mysql.TypeTimestamp, + } + // test for min int64 handle + idxValue, _, err := index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(value)}, math.MinInt64, nil) + c.Assert(err, IsNil) + idx := searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + + // Test for max int64 handle. + idxValue, _, err = index.GenIndexKey(ctx.GetSessionVars().StmtCtx, []types.Datum{types.NewDatum(value)}, math.MaxInt64, nil) + c.Assert(err, IsNil) + idx = searchLessEqualIdx(valueList, idxValue) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + } +} + +func searchLessEqualIdx(valueList [][]byte, value []byte) int { + idx := -1 + for i, v := range valueList { + if bytes.Compare(value, v) >= 0 { + idx = i + continue + } + break + } + return idx +} diff --git a/go.mod b/go.mod index 70a6f40911b88..f533869296d92 100644 --- a/go.mod +++ b/go.mod @@ -73,3 +73,5 @@ require ( sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) + +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190509070214-ba6b0df5609f diff --git a/go.sum b/go.sum index 288f6e6d080d2..4f0cacbf76fe9 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/crazycs520/parser v0.0.0-20190509070214-ba6b0df5609f h1:zl3+ArJtnQlr7uj1KIv5d6hQXTSw0cRSSRpwzPGw17k= +github.com/crazycs520/parser v0.0.0-20190509070214-ba6b0df5609f/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index a918ed17ff218..3efa74cbd46a7 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -481,8 +481,11 @@ type LoadStats struct { type SplitIndexRegion struct { baseSchemaProducer - Table table.Table + TableInfo *model.TableInfo IndexInfo *model.IndexInfo + Min []types.Datum + Max []types.Datum + Num int ValueLists [][]types.Datum } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 118b41c8bd9d3..cc7c2d2cbdea1 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" + "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/ranger" ) @@ -1607,44 +1608,97 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { return p } +const maxSplitRegionNum = 1000 + func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Plan, error) { tblInfo := node.Table.TableInfo - indexInfo := tblInfo.FindIndexByName(strings.ToLower(node.IndexName)) + indexInfo := tblInfo.FindIndexByName(node.IndexName.L) if indexInfo == nil { return nil, ErrKeyDoesNotExist.GenWithStackByArgs(node.IndexName, tblInfo.Name) } + mockTablePlan := LogicalTableDual{}.Init(b.ctx) + schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo) + mockTablePlan.SetSchema(schema) - indexValues := make([][]types.Datum, 0, len(node.ValueLists)) - for i, valuesItem := range node.ValueLists { - if len(valuesItem) > len(indexInfo.Columns) { - return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) - } - valueList := make([]types.Datum, 0, len(valuesItem)) + checkValue := func(valuesItem []ast.ExprNode) ([]types.Datum, error) { + values := make([]types.Datum, 0, len(valuesItem)) for j, valueItem := range valuesItem { - x, ok := valueItem.(*driver.ValueExpr) + var expr expression.Expression + var err error + switch x := valueItem.(type) { + case *driver.ValueExpr: + expr = &expression.Constant{ + Value: x.Datum, + RetType: &x.Type, + } + default: + expr, _, err = b.rewrite(valueItem, mockTablePlan, nil, true) + if err != nil { + return nil, err + } + } + constant, ok := expr.(*expression.Constant) if !ok { return nil, errors.New("expect constant values") } + value, err := constant.Eval(chunk.Row{}) + if err != nil { + return nil, err + } colOffset := indexInfo.Columns[j].Offset - value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) + value, err = value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) if err != nil { return nil, err } + values = append(values, value) + } + return values, nil + } + p := &SplitIndexRegion{ + TableInfo: tblInfo, + IndexInfo: indexInfo, + } + if len(node.SplitOpt.ValueLists) > 0 { + indexValues := make([][]types.Datum, 0, len(node.SplitOpt.ValueLists)) + for i, valuesItem := range node.SplitOpt.ValueLists { + if len(valuesItem) > len(indexInfo.Columns) { + return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) + } + values, err := checkValue(valuesItem) + if err != nil { + return nil, err + } + indexValues = append(indexValues, values) + } + p.ValueLists = indexValues + return p, nil + } - valueList = append(valueList, value) + checkMinMaxValue := func(valuesItem []ast.ExprNode, name string) ([]types.Datum, error) { + if len(valuesItem) == 0 { + return nil, errors.Errorf("Split index region `%v` %s value count should more than 0", indexInfo.Name, name) } - indexValues = append(indexValues, valueList) + if len(valuesItem) > len(indexInfo.Columns) { + return nil, errors.Errorf("Split index region `%v` Column count doesn't match value count at %v", indexInfo.Name, name) + } + return checkValue(valuesItem) } - tableInPlan, ok := b.is.TableByID(tblInfo.ID) - if !ok { - return nil, errors.Errorf("Can't get table %s.", tblInfo.Name.O) + minValues, err := checkMinMaxValue(node.SplitOpt.Min, "min") + if err != nil { + return nil, err } - return &SplitIndexRegion{ - Table: tableInPlan, - IndexInfo: indexInfo, - ValueLists: indexValues, - }, nil + maxValue, err := checkMinMaxValue(node.SplitOpt.Max, "max") + if err != nil { + return nil, err + } + p.Min = minValues + p.Max = maxValue + if node.SplitOpt.Num > maxSplitRegionNum { + return nil, errors.Errorf("Split index region num is exceed the limit %v", maxSplitRegionNum) + } + p.Num = int(node.SplitOpt.Num) + return p, nil } func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) { From bd39a3b8a7a172497482d25dfbf4c94eefa1607d Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 9 May 2019 21:18:15 +0800 Subject: [PATCH 02/21] refine code --- executor/split.go | 21 +++++++++++++++------ executor/split_test.go | 40 +++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/executor/split.go b/executor/split.go index d3c316079b04b..f9205662cf016 100644 --- a/executor/split.go +++ b/executor/split.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" @@ -87,7 +88,16 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e } func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { - idxKeys := make([][]byte, 0, len(e.valueLists)) + var idxKeys [][]byte + if e.num > 0 { + idxKeys = make([][]byte, 0, e.num) + } else { + idxKeys = make([][]byte, 0, len(e.valueLists)+1) + } + // Split in the start of the index key. + startIdxKey := tablecodec.EncodeTableIndexPrefix(e.tableInfo.ID, e.indexInfo.ID) + idxKeys = append(idxKeys, startIdxKey) + index := tables.NewIndex(e.tableInfo.ID, e.tableInfo, e.indexInfo) if len(e.valueLists) > 0 { for _, v := range e.valueLists { @@ -110,8 +120,7 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { if bytes.Compare(minIdxKey, maxIdxKey) >= 0 { return nil, errors.Errorf("Split index region `%v` min value %v should less than the max value %v", e.indexInfo.Name, e.min, e.max) } - idxKeys = make([][]byte, 0, e.num+1) - return getValuesList(minIdxKey, maxIdxKey, e.num), nil + return getValuesList(minIdxKey, maxIdxKey, e.num, idxKeys), nil } func longestCommonPrefixLen(s1, s2 []byte) int { @@ -164,7 +173,7 @@ func getDiffBytesValue(startIdx int, min, max []byte) uint64 { return diffValue } -func getValuesList(min, max []byte, num int) [][]byte { +func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { startIdx := longestCommonPrefixLen(min, max) diffValue := getDiffBytesValue(startIdx, min, max) step := diffValue / uint64(num) @@ -179,8 +188,8 @@ func getValuesList(min, max []byte, num int) [][]byte { startValue = append(startValue, 0) } startV := binary.BigEndian.Uint64(startValue) - valuesList := make([][]byte, 0, num+1) - valuesList = append(valuesList, min) + // To get `num` regions, only need to split `num-1` idx keys. + num-- tmp := make([]byte, 8) for i := 0; i < num; i++ { value := make([]byte, 0, startIdx+8) diff --git a/executor/split_test.go b/executor/split_test.go index 50acfc065ad18..ca692d84f4265 100644 --- a/executor/split_test.go +++ b/executor/split_test.go @@ -111,8 +111,7 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // Test for int index. // range is 0 ~ 100, and split into 10 region. // So 10 regions range is like below: - // region0: -inf ~ 0 - // region1: 0 ~ 10 + // region1: -inf ~ 10 // region2: 10 ~ 20 // region3: 20 ~ 30 // region4: 30 ~ 40 @@ -133,20 +132,19 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { } valueList, err := e.getSplitIdxKeys() c.Assert(err, IsNil) - c.Assert(len(valueList), Equals, e.num+1) + c.Assert(len(valueList), Equals, e.num) cases := []struct { value int lessEqualIdx int }{ - {-1, -1}, + {-1, 0}, {0, 0}, {1, 0}, {10, 1}, {11, 1}, {20, 2}, {21, 2}, - {21, 2}, {31, 3}, {41, 4}, {51, 5}, @@ -154,8 +152,8 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { {71, 7}, {81, 8}, {91, 9}, - {100, 10}, - {1000, 10}, + {100, 9}, + {1000, 9}, } index := tables.NewIndex(tbInfo.ID, tbInfo, idxInfo) @@ -175,12 +173,12 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // Test for varchar index. // range is a ~ z, and split into 26 region. // So 26 regions range is like below: - // region0: -inf ~ a - // region1: a ~ b + // region1: -inf ~ b + // region2: b ~ c // . // . // . - // region26: z ~ +inf + // region26: y ~ +inf e.min = []types.Datum{types.NewDatum("a")} e.max = []types.Datum{types.NewDatum("z")} e.num = 26 @@ -189,21 +187,21 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { valueList, err = e.getSplitIdxKeys() c.Assert(err, IsNil) - c.Assert(len(valueList), Equals, e.num+1) + c.Assert(len(valueList), Equals, e.num) cases2 := []struct { value string lessEqualIdx int }{ - {"", -1}, + {"", 0}, {"a", 0}, {"abcde", 0}, {"b", 1}, {"bzzzz", 1}, {"c", 2}, {"czzzz", 2}, - {"z", 26}, - {"zabcd", 26}, + {"z", 25}, + {"zabcd", 25}, } for _, ca := range cases2 { @@ -223,12 +221,12 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // Test for timestamp index. // range is 2010-01-01 00:00:00 ~ 2020-01-01 00:00:00, and split into 10 region. // So 10 regions range is like below: - // region0: -inf ~ 2010-01-01 00:00:00 - // region1: 2010-01-01 00:00:00 ~ 2011-01-01 00:00:00 + // region1: -inf ~ 2011-01-01 00:00:00 + // region2: 2011-01-01 00:00:00 ~ 2012-01-01 00:00:00 // . // . // . - // region10: 2020-01-01 00:00:00 ~ +inf + // region10: 2019-01-01 00:00:00 ~ +inf minTime := types.Time{ Time: types.FromDate(2010, 1, 1, 0, 0, 0, 0), Type: mysql.TypeTimestamp, @@ -246,13 +244,13 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { valueList, err = e.getSplitIdxKeys() c.Assert(err, IsNil) - c.Assert(len(valueList), Equals, e.num+1) + c.Assert(len(valueList), Equals, e.num) cases3 := []struct { value types.MysqlTime lessEqualIdx int }{ - {types.FromDate(2009, 11, 20, 12, 50, 59, 0), -1}, + {types.FromDate(2009, 11, 20, 12, 50, 59, 0), 0}, {types.FromDate(2010, 1, 1, 0, 0, 0, 0), 0}, {types.FromDate(2011, 12, 31, 23, 59, 59, 0), 1}, {types.FromDate(2011, 2, 1, 0, 0, 0, 0), 1}, @@ -264,8 +262,8 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { {types.FromDate(2017, 9, 1, 0, 0, 0, 0), 7}, {types.FromDate(2018, 10, 1, 0, 0, 0, 0), 8}, {types.FromDate(2019, 11, 1, 0, 0, 0, 0), 9}, - {types.FromDate(2020, 12, 1, 0, 0, 0, 0), 10}, - {types.FromDate(2030, 12, 1, 0, 0, 0, 0), 10}, + {types.FromDate(2020, 12, 1, 0, 0, 0, 0), 9}, + {types.FromDate(2030, 12, 1, 0, 0, 0, 0), 9}, } for _, ca := range cases3 { From 5b5f87198bb62d5ff4bac2eda41b6ec383124a4a Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 20 May 2019 11:56:09 +0800 Subject: [PATCH 03/21] update go.mod for parsre --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d3737a26d0347..0d621104a9ea4 100644 --- a/go.mod +++ b/go.mod @@ -74,4 +74,4 @@ require ( sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190509070214-ba6b0df5609f +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190520031308-8ef5b9c71c22 diff --git a/go.sum b/go.sum index e82678dbfd0d8..283831ae13d26 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/crazycs520/parser v0.0.0-20190509070214-ba6b0df5609f h1:zl3+ArJtnQlr7uj1KIv5d6hQXTSw0cRSSRpwzPGw17k= -github.com/crazycs520/parser v0.0.0-20190509070214-ba6b0df5609f/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= +github.com/crazycs520/parser v0.0.0-20190520031308-8ef5b9c71c22 h1:RSCoPexphqCHE0FvVTXl7yvP/aKSQiYGCysZWFaoeKc= +github.com/crazycs520/parser v0.0.0-20190520031308-8ef5b9c71c22/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= @@ -163,8 +163,6 @@ github.com/pingcap/kvproto v0.0.0-20190517030054-ff2e03f6fdfe/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190517034656-b210fe7db725 h1:dcH76qohLDic7h5O0+tPWT6ycjA15fkpSOyaqCGbzys= -github.com/pingcap/parser v0.0.0-20190517034656-b210fe7db725/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From 553c9bece39d24e12f5834fcd970174e24955247 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 20 May 2019 21:06:38 +0800 Subject: [PATCH 04/21] update go.mod for parsre --- executor/executor.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- planner/core/planbuilder.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index 16bf5746049dd..b7f52e3085001 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1388,7 +1388,7 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.InShowWarning = true sc.SetWarnings(vars.StmtCtx.GetWarnings()) } - case *ast.SplitIndexRegionStmt: + case *ast.SplitRegionStmt: sc.IgnoreTruncate = false sc.IgnoreZeroInDate = true sc.AllowInvalidDate = vars.SQLMode.HasAllowInvalidDatesMode() diff --git a/go.mod b/go.mod index 0d621104a9ea4..22bcec045bf3d 100644 --- a/go.mod +++ b/go.mod @@ -74,4 +74,4 @@ require ( sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190520031308-8ef5b9c71c22 +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190520125827-10bf17237e4a diff --git a/go.sum b/go.sum index 283831ae13d26..9b683129eaa55 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/crazycs520/parser v0.0.0-20190520031308-8ef5b9c71c22 h1:RSCoPexphqCHE0FvVTXl7yvP/aKSQiYGCysZWFaoeKc= -github.com/crazycs520/parser v0.0.0-20190520031308-8ef5b9c71c22/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= +github.com/crazycs520/parser v0.0.0-20190520125827-10bf17237e4a h1:AHsi/YPtUbo3qHOGrpjxaUTroq1Hh31BaxM+LUIRFvQ= +github.com/crazycs520/parser v0.0.0-20190520125827-10bf17237e4a/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 873b98bc7b13e..3a106830c2be5 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -280,7 +280,7 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) { return b.buildDropBindPlan(x) case *ast.ChangeStmt: return b.buildChange(x) - case *ast.SplitIndexRegionStmt: + case *ast.SplitRegionStmt: return b.buildSplitIndexRegion(x) } return nil, ErrUnsupportedType.GenWithStack("Unsupported type %T", node) @@ -1610,7 +1610,7 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { const maxSplitRegionNum = 1000 -func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Plan, error) { +func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitRegionStmt) (Plan, error) { tblInfo := node.Table.TableInfo indexInfo := tblInfo.FindIndexByName(node.IndexName.L) if indexInfo == nil { From cdb0e6ee7075aeb4be0092aed7a301c35fe1386d Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 20 May 2019 22:24:56 +0800 Subject: [PATCH 05/21] refactor code --- executor/builder.go | 4 ++-- planner/core/common_plans.go | 2 +- planner/core/planbuilder.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index a60c2009e683b..89e4813b10f56 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -194,7 +194,7 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildWindow(v) case *plannercore.SQLBindPlan: return b.buildSQLBindExec(v) - case *plannercore.SplitIndexRegion: + case *plannercore.SplitRegion: return b.buildSplitIndexRegion(v) default: if mp, ok := p.(MockPhysicalPlan); ok { @@ -1255,7 +1255,7 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo return e } -func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitIndexRegion) Executor { +func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitRegion) Executor { base := newBaseExecutor(b.ctx, nil, v.ExplainID()) base.initCap = chunk.ZeroCapacity return &SplitIndexRegionExec{ diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 3efa74cbd46a7..44a34859ddc95 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -478,7 +478,7 @@ type LoadStats struct { } // SplitIndexRegion represents a split index regions plan. -type SplitIndexRegion struct { +type SplitRegion struct { baseSchemaProducer TableInfo *model.TableInfo diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 3a106830c2be5..53e4223302f4e 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -281,7 +281,7 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) { case *ast.ChangeStmt: return b.buildChange(x) case *ast.SplitRegionStmt: - return b.buildSplitIndexRegion(x) + return b.buildSplitRegion(x) } return nil, ErrUnsupportedType.GenWithStack("Unsupported type %T", node) } @@ -1610,7 +1610,7 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { const maxSplitRegionNum = 1000 -func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitRegionStmt) (Plan, error) { +func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) { tblInfo := node.Table.TableInfo indexInfo := tblInfo.FindIndexByName(node.IndexName.L) if indexInfo == nil { @@ -1654,7 +1654,7 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitRegionStmt) (Plan, er } return values, nil } - p := &SplitIndexRegion{ + p := &SplitRegion{ TableInfo: tblInfo, IndexInfo: indexInfo, } From 3668f80c9184e88b0373c1bc2b779698a3253f36 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 4 Jun 2019 18:02:59 +0800 Subject: [PATCH 06/21] add comment --- executor/split.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/executor/split.go b/executor/split.go index f9205662cf016..21c152020c94a 100644 --- a/executor/split.go +++ b/executor/split.go @@ -99,6 +99,7 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { idxKeys = append(idxKeys, startIdxKey) index := tables.NewIndex(e.tableInfo.ID, e.tableInfo, e.indexInfo) + // Split index regions by user specified value lists. if len(e.valueLists) > 0 { for _, v := range e.valueLists { idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, v, math.MinInt64, nil) @@ -109,6 +110,7 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { } return idxKeys, nil } + // Split index regions by min, max value and calculate the step by (max - min)/num. minIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.min, math.MinInt64, nil) if err != nil { return nil, err @@ -123,6 +125,7 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { return getValuesList(minIdxKey, maxIdxKey, e.num, idxKeys), nil } +// longestCommonPrefixLen gets the longest common prefix byte length. func longestCommonPrefixLen(s1, s2 []byte) int { l := len(s1) if len(s2) < len(s1) { @@ -137,6 +140,15 @@ func longestCommonPrefixLen(s1, s2 []byte) int { return i } +// getDiffBytesValue gets the diff-value from the `startIdx`. Normally, `startIdx` is the longest common prefix byte length. +// eg: min: [10,1, 2,3,4,5] +// max: [10,10,9,8,7,6,5,4] +// startIdx: 1 +// the diff bytes is [10,9,8,7,6,5,4] +// - [1, 2,3,4,5] +// = [9, 7,5,3,1,5,4] +// I need 8 diff-bytes to convert(decode) a uint64 value. So append 0xff to complete 8 bytes. +// so the return uint64 value is binary.BigEndian.Uint64([]byte{9,7,5,3,1,5,4,255}). func getDiffBytesValue(startIdx int, min, max []byte) uint64 { l := len(min) if len(max) < len(min) { @@ -173,6 +185,11 @@ func getDiffBytesValue(startIdx int, min, max []byte) uint64 { return diffValue } +// getValuesList use to get `num` values between min and max value. +// To Simplify the explain, suppose min and max value type is int64, and min=0, max=100, num=10, +// then calculate the step=(max-min)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. +// then the function will return [10,20,30,40,50,60,70,80,90]. +// The difference is the max,min value type is []byte, So I use getDiffBytesValue to calculate the (max-min) value. func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { startIdx := longestCommonPrefixLen(min, max) diffValue := getDiffBytesValue(startIdx, min, max) From 255845711c4c1e2ece4352f747f0ba0c66d0d60c Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 4 Jun 2019 19:59:49 +0800 Subject: [PATCH 07/21] refine code --- executor/split.go | 67 +++++++++++++----------------------------- executor/split_test.go | 4 +-- 2 files changed, 23 insertions(+), 48 deletions(-) diff --git a/executor/split.go b/executor/split.go index 21c152020c94a..37d7f8ae68e34 100644 --- a/executor/split.go +++ b/executor/split.go @@ -140,60 +140,36 @@ func longestCommonPrefixLen(s1, s2 []byte) int { return i } -// getDiffBytesValue gets the diff-value from the `startIdx`. Normally, `startIdx` is the longest common prefix byte length. -// eg: min: [10,1, 2,3,4,5] -// max: [10,10,9,8,7,6,5,4] -// startIdx: 1 -// the diff bytes is [10,9,8,7,6,5,4] -// - [1, 2,3,4,5] -// = [9, 7,5,3,1,5,4] -// I need 8 diff-bytes to convert(decode) a uint64 value. So append 0xff to complete 8 bytes. -// so the return uint64 value is binary.BigEndian.Uint64([]byte{9,7,5,3,1,5,4,255}). -func getDiffBytesValue(startIdx int, min, max []byte) uint64 { - l := len(min) - if len(max) < len(min) { - l = len(max) - } - if l-startIdx > 8 { - l = startIdx + 8 - } - diff := make([]byte, 0, 8) - for i := startIdx; i < l; i++ { - diff = append(diff, max[i]-min[i]) - } - if len(max) > l { - for i := l; i < len(max); i++ { - diff = append(diff, max[i]) - if len(diff) >= 8 { - break - } - } - } - if len(min) > l { - for i := l; i < len(min); i++ { - diff = append(diff, 0xff-min[i]) - if len(diff) >= 8 { - break - } - } - } +// getStepValue gets the step of between the min and max value. step = (max-min)/num. +// convert byte slice to uint64 first. +func getStepValue(min, max []byte, num int) uint64 { + minUint := getUint64FromBytes(min, 0) + maxUint := getUint64FromBytes(max, 0xff) + return (maxUint - minUint) / uint64(num) +} - for i := len(diff); i < 8; i++ { - diff = append(diff, 0xff) +// getUint64FromBytes gets a uint64 from the `bs` byte slice. +// If len(bs) < 8, then padding with `pad`. +func getUint64FromBytes(bs []byte, pad byte) uint64 { + buf := bs + if len(buf) < 8 { + buf = make([]byte, 0, 8) + buf = append(buf, bs...) + for i := len(buf); i < 8; i++ { + buf = append(buf, pad) + } } - diffValue := binary.BigEndian.Uint64(diff) - return diffValue + return binary.BigEndian.Uint64(buf) } // getValuesList use to get `num` values between min and max value. // To Simplify the explain, suppose min and max value type is int64, and min=0, max=100, num=10, // then calculate the step=(max-min)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. // then the function will return [10,20,30,40,50,60,70,80,90]. -// The difference is the max,min value type is []byte, So I use getDiffBytesValue to calculate the (max-min) value. +// The difference is the max,min value type is []byte, So I use getUint64FromBytes to convert []byte to uint64. func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { startIdx := longestCommonPrefixLen(min, max) - diffValue := getDiffBytesValue(startIdx, min, max) - step := diffValue / uint64(num) + step := getStepValue(min[startIdx:], max[startIdx:], num) startValueTemp := min[startIdx:] if len(startValueTemp) > 8 { @@ -206,9 +182,8 @@ func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { } startV := binary.BigEndian.Uint64(startValue) // To get `num` regions, only need to split `num-1` idx keys. - num-- tmp := make([]byte, 8) - for i := 0; i < num; i++ { + for i := 0; i < num-1; i++ { value := make([]byte, 0, startIdx+8) value = append(value, min[:startIdx]...) startV += step diff --git a/executor/split_test.go b/executor/split_test.go index ca692d84f4265..84752b64fc93a 100644 --- a/executor/split_test.go +++ b/executor/split_test.go @@ -60,7 +60,7 @@ func (s *testSplitIndex) TestLongestCommonPrefixLen(c *C) { } } -func (s *testSplitIndex) TestGetDiffBytesValue(c *C) { +func (s *testSplitIndex) TestgetStepValue(c *C) { cases := []struct { min []byte max []byte @@ -79,7 +79,7 @@ func (s *testSplitIndex) TestGetDiffBytesValue(c *C) { for _, ca := range cases { l0 := longestCommonPrefixLen(ca.min, ca.max) c.Assert(l0, Equals, ca.l) - v0 := getDiffBytesValue(l0, ca.min, ca.max) + v0 := getStepValue(ca.min[l0:], ca.max[l0:], 1) c.Assert(v0, Equals, ca.v) } } From 078b8f4693426821b6da2c224e755e49fe845364 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 4 Jun 2019 20:06:52 +0800 Subject: [PATCH 08/21] address comment --- executor/split.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/executor/split.go b/executor/split.go index 37d7f8ae68e34..c95e2bf5ebb10 100644 --- a/executor/split.go +++ b/executor/split.go @@ -162,16 +162,16 @@ func getUint64FromBytes(bs []byte, pad byte) uint64 { return binary.BigEndian.Uint64(buf) } -// getValuesList use to get `num` values between min and max value. +// getValuesList is used to get `num` values between min and max value. // To Simplify the explain, suppose min and max value type is int64, and min=0, max=100, num=10, // then calculate the step=(max-min)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. // then the function will return [10,20,30,40,50,60,70,80,90]. // The difference is the max,min value type is []byte, So I use getUint64FromBytes to convert []byte to uint64. func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { - startIdx := longestCommonPrefixLen(min, max) - step := getStepValue(min[startIdx:], max[startIdx:], num) + commonPrefixIdx := longestCommonPrefixLen(min, max) + step := getStepValue(min[commonPrefixIdx:], max[commonPrefixIdx:], num) - startValueTemp := min[startIdx:] + startValueTemp := min[commonPrefixIdx:] if len(startValueTemp) > 8 { startValueTemp = startValueTemp[:8] } @@ -184,8 +184,8 @@ func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { // To get `num` regions, only need to split `num-1` idx keys. tmp := make([]byte, 8) for i := 0; i < num-1; i++ { - value := make([]byte, 0, startIdx+8) - value = append(value, min[:startIdx]...) + value := make([]byte, 0, commonPrefixIdx+8) + value = append(value, min[:commonPrefixIdx]...) startV += step binary.BigEndian.PutUint64(tmp, startV) value = append(value, tmp...) From 208a15fc3deb72b6e349d813977e94078fe0009f Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 4 Jun 2019 20:11:19 +0800 Subject: [PATCH 09/21] refine code --- executor/split.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/executor/split.go b/executor/split.go index c95e2bf5ebb10..24b92eefb3ae1 100644 --- a/executor/split.go +++ b/executor/split.go @@ -170,25 +170,15 @@ func getUint64FromBytes(bs []byte, pad byte) uint64 { func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { commonPrefixIdx := longestCommonPrefixLen(min, max) step := getStepValue(min[commonPrefixIdx:], max[commonPrefixIdx:], num) - - startValueTemp := min[commonPrefixIdx:] - if len(startValueTemp) > 8 { - startValueTemp = startValueTemp[:8] - } - startValue := make([]byte, 0, 8) - startValue = append(startValue, startValueTemp...) - for i := len(startValue); i < 8; i++ { - startValue = append(startValue, 0) - } - startV := binary.BigEndian.Uint64(startValue) + startV := getUint64FromBytes(min[commonPrefixIdx:], 0) // To get `num` regions, only need to split `num-1` idx keys. - tmp := make([]byte, 8) + buf := make([]byte, 8) for i := 0; i < num-1; i++ { value := make([]byte, 0, commonPrefixIdx+8) value = append(value, min[:commonPrefixIdx]...) startV += step - binary.BigEndian.PutUint64(tmp, startV) - value = append(value, tmp...) + binary.BigEndian.PutUint64(buf, startV) + value = append(value, buf...) valuesList = append(valuesList, value) } return valuesList From ad2af9f5e30fe5a582cca1abad19f4d2e2bd9967 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 4 Jun 2019 20:14:35 +0800 Subject: [PATCH 10/21] Address comment --- executor/split.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/split.go b/executor/split.go index 24b92eefb3ae1..383d4d3e413e9 100644 --- a/executor/split.go +++ b/executor/split.go @@ -166,7 +166,7 @@ func getUint64FromBytes(bs []byte, pad byte) uint64 { // To Simplify the explain, suppose min and max value type is int64, and min=0, max=100, num=10, // then calculate the step=(max-min)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. // then the function will return [10,20,30,40,50,60,70,80,90]. -// The difference is the max,min value type is []byte, So I use getUint64FromBytes to convert []byte to uint64. +// The difference is the value type of max,min is []byte, So I use getUint64FromBytes to convert []byte to uint64. func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { commonPrefixIdx := longestCommonPrefixLen(min, max) step := getStepValue(min[commonPrefixIdx:], max[commonPrefixIdx:], num) From 90d5bef6e6622b1bf6717fdc475da5a29cb9f065 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 4 Jun 2019 21:43:10 +0800 Subject: [PATCH 11/21] address comment --- executor/split_test.go | 6 +++--- planner/core/planbuilder.go | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/executor/split_test.go b/executor/split_test.go index 84752b64fc93a..2ab14e0fb5d9a 100644 --- a/executor/split_test.go +++ b/executor/split_test.go @@ -77,9 +77,9 @@ func (s *testSplitIndex) TestgetStepValue(c *C) { } for _, ca := range cases { - l0 := longestCommonPrefixLen(ca.min, ca.max) - c.Assert(l0, Equals, ca.l) - v0 := getStepValue(ca.min[l0:], ca.max[l0:], 1) + l := longestCommonPrefixLen(ca.min, ca.max) + c.Assert(l, Equals, ca.l) + v0 := getStepValue(ca.min[l:], ca.max[l:], 1) c.Assert(v0, Equals, ca.v) } } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 53e4223302f4e..18066a9eee031 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1620,7 +1620,7 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo) mockTablePlan.SetSchema(schema) - checkValue := func(valuesItem []ast.ExprNode) ([]types.Datum, error) { + convertValue2ColumnType := func(valuesItem []ast.ExprNode) ([]types.Datum, error) { values := make([]types.Datum, 0, len(valuesItem)) for j, valueItem := range valuesItem { var expr expression.Expression @@ -1658,13 +1658,14 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) TableInfo: tblInfo, IndexInfo: indexInfo, } + // Split index regions by user specified value lists. if len(node.SplitOpt.ValueLists) > 0 { indexValues := make([][]types.Datum, 0, len(node.SplitOpt.ValueLists)) for i, valuesItem := range node.SplitOpt.ValueLists { if len(valuesItem) > len(indexInfo.Columns) { return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) } - values, err := checkValue(valuesItem) + values, err := convertValue2ColumnType(valuesItem) if err != nil { return nil, err } @@ -1674,6 +1675,7 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) return p, nil } + // Split index regions by min, max value. checkMinMaxValue := func(valuesItem []ast.ExprNode, name string) ([]types.Datum, error) { if len(valuesItem) == 0 { return nil, errors.Errorf("Split index region `%v` %s value count should more than 0", indexInfo.Name, name) @@ -1681,18 +1683,18 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) if len(valuesItem) > len(indexInfo.Columns) { return nil, errors.Errorf("Split index region `%v` Column count doesn't match value count at %v", indexInfo.Name, name) } - return checkValue(valuesItem) + return convertValue2ColumnType(valuesItem) } minValues, err := checkMinMaxValue(node.SplitOpt.Min, "min") if err != nil { return nil, err } - maxValue, err := checkMinMaxValue(node.SplitOpt.Max, "max") + maxValues, err := checkMinMaxValue(node.SplitOpt.Max, "max") if err != nil { return nil, err } p.Min = minValues - p.Max = maxValue + p.Max = maxValues if node.SplitOpt.Num > maxSplitRegionNum { return nil, errors.Errorf("Split index region num is exceed the limit %v", maxSplitRegionNum) From ccf2e8e3eaeb2f1c7e14904dc03c0138858331fb Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 5 Jun 2019 13:08:38 +0800 Subject: [PATCH 12/21] fix region num 0 bug and add test --- executor/executor_test.go | 5 +++++ planner/core/planbuilder.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/executor/executor_test.go b/executor/executor_test.go index 87ae21ad979ee..731a8fb0cb1eb 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3822,6 +3822,11 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 10000`) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "Split index region num is exceed the limit 1000") + + // Check pre-split region num 0 is invalid. + _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 0`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split index region num should more than 0") } func (s *testSuite) TestIssue10435(c *C) { diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 59071a77a9c58..e1d50a5ad9425 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1710,6 +1710,8 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) if node.SplitOpt.Num > maxSplitRegionNum { return nil, errors.Errorf("Split index region num is exceed the limit %v", maxSplitRegionNum) + } else if node.SplitOpt.Num < 1 { + return nil, errors.Errorf("Split index region num should more than 0") } p.Num = int(node.SplitOpt.Num) return p, nil From 6ec298f4f1c7f375f7f240834bda431f6ddcc1ab Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 5 Jun 2019 13:31:01 +0800 Subject: [PATCH 13/21] use lower and upper instead of min/max --- executor/builder.go | 4 ++-- executor/executor_test.go | 6 ++--- executor/split.go | 44 ++++++++++++++++++------------------ executor/split_test.go | 28 +++++++++++------------ go.mod | 2 +- go.sum | 2 ++ planner/core/common_plans.go | 4 ++-- planner/core/planbuilder.go | 12 +++++----- 8 files changed, 52 insertions(+), 50 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index 4b3b0925ca0cb..a55bad1f91170 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1268,8 +1268,8 @@ func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitRegion) Exec baseExecutor: base, tableInfo: v.TableInfo, indexInfo: v.IndexInfo, - min: v.Min, - max: v.Max, + lower: v.Lower, + upper: v.Upper, num: v.Num, valueLists: v.ValueLists, } diff --git a/executor/executor_test.go b/executor/executor_test.go index 731a8fb0cb1eb..900339192e5bb 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3806,17 +3806,17 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { tk.MustExec(`split table t index idx1 between (0) and (1000000000) regions 10`) _, err = tk.Exec(`split table t index idx1 between (2) and (1) regions 10`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region `idx1` min value [{1 0 0 0 2 [] }] should less than the max value [{1 0 0 0 1 [] }]") + c.Assert(err.Error(), Equals, "Split index region `idx1` lower value [{1 0 0 0 2 [] }] should less than the upper value [{1 0 0 0 1 [] }]") // Check min value is invalid. _, err = tk.Exec(`split table t index idx1 between () and (1) regions 10`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region `idx1` min value count should more than 0") + c.Assert(err.Error(), Equals, "Split index region `idx1` lower value count should more than 0") // Check max value is invalid. _, err = tk.Exec(`split table t index idx1 between (1) and () regions 10`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region `idx1` max value count should more than 0") + c.Assert(err.Error(), Equals, "Split index region `idx1` upper value count should more than 0") // Check pre-split region num is too large. _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 10000`) diff --git a/executor/split.go b/executor/split.go index 383d4d3e413e9..09bd9a4672c4d 100644 --- a/executor/split.go +++ b/executor/split.go @@ -36,8 +36,8 @@ type SplitIndexRegionExec struct { tableInfo *model.TableInfo indexInfo *model.IndexInfo - min []types.Datum - max []types.Datum + lower []types.Datum + upper []types.Datum num int valueLists [][]types.Datum } @@ -110,19 +110,19 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { } return idxKeys, nil } - // Split index regions by min, max value and calculate the step by (max - min)/num. - minIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.min, math.MinInt64, nil) + // Split index regions by lower, upper value and calculate the step by (upper - lower)/num. + lowerIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.lower, math.MinInt64, nil) if err != nil { return nil, err } - maxIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.max, math.MinInt64, nil) + upperIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.upper, math.MinInt64, nil) if err != nil { return nil, err } - if bytes.Compare(minIdxKey, maxIdxKey) >= 0 { - return nil, errors.Errorf("Split index region `%v` min value %v should less than the max value %v", e.indexInfo.Name, e.min, e.max) + if bytes.Compare(lowerIdxKey, upperIdxKey) >= 0 { + return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper) } - return getValuesList(minIdxKey, maxIdxKey, e.num, idxKeys), nil + return getValuesList(lowerIdxKey, upperIdxKey, e.num, idxKeys), nil } // longestCommonPrefixLen gets the longest common prefix byte length. @@ -140,12 +140,12 @@ func longestCommonPrefixLen(s1, s2 []byte) int { return i } -// getStepValue gets the step of between the min and max value. step = (max-min)/num. +// getStepValue gets the step of between the lower and upper value. step = (upper-lower)/num. // convert byte slice to uint64 first. -func getStepValue(min, max []byte, num int) uint64 { - minUint := getUint64FromBytes(min, 0) - maxUint := getUint64FromBytes(max, 0xff) - return (maxUint - minUint) / uint64(num) +func getStepValue(lower, upper []byte, num int) uint64 { + lowerUint := getUint64FromBytes(lower, 0) + upperUint := getUint64FromBytes(upper, 0xff) + return (upperUint - lowerUint) / uint64(num) } // getUint64FromBytes gets a uint64 from the `bs` byte slice. @@ -162,20 +162,20 @@ func getUint64FromBytes(bs []byte, pad byte) uint64 { return binary.BigEndian.Uint64(buf) } -// getValuesList is used to get `num` values between min and max value. -// To Simplify the explain, suppose min and max value type is int64, and min=0, max=100, num=10, -// then calculate the step=(max-min)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. +// getValuesList is used to get `num` values between lower and upper value. +// To Simplify the explain, suppose lower and upper value type is int64, and lower=0, upper=100, num=10, +// then calculate the step=(upper-lower)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. // then the function will return [10,20,30,40,50,60,70,80,90]. -// The difference is the value type of max,min is []byte, So I use getUint64FromBytes to convert []byte to uint64. -func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte { - commonPrefixIdx := longestCommonPrefixLen(min, max) - step := getStepValue(min[commonPrefixIdx:], max[commonPrefixIdx:], num) - startV := getUint64FromBytes(min[commonPrefixIdx:], 0) +// The difference is the value type of upper,lower is []byte, So I use getUint64FromBytes to convert []byte to uint64. +func getValuesList(lower, upper []byte, num int, valuesList [][]byte) [][]byte { + commonPrefixIdx := longestCommonPrefixLen(lower, upper) + step := getStepValue(lower[commonPrefixIdx:], upper[commonPrefixIdx:], num) + startV := getUint64FromBytes(lower[commonPrefixIdx:], 0) // To get `num` regions, only need to split `num-1` idx keys. buf := make([]byte, 8) for i := 0; i < num-1; i++ { value := make([]byte, 0, commonPrefixIdx+8) - value = append(value, min[:commonPrefixIdx]...) + value = append(value, lower[:commonPrefixIdx]...) startV += step binary.BigEndian.PutUint64(buf, startV) value = append(value, buf...) diff --git a/executor/split_test.go b/executor/split_test.go index 2ab14e0fb5d9a..da4545088a08b 100644 --- a/executor/split_test.go +++ b/executor/split_test.go @@ -62,10 +62,10 @@ func (s *testSplitIndex) TestLongestCommonPrefixLen(c *C) { func (s *testSplitIndex) TestgetStepValue(c *C) { cases := []struct { - min []byte - max []byte - l int - v uint64 + lower []byte + upper []byte + l int + v uint64 }{ {[]byte{}, []byte{}, 0, math.MaxUint64}, {[]byte{0}, []byte{128}, 0, binary.BigEndian.Uint64([]byte{128, 255, 255, 255, 255, 255, 255, 255})}, @@ -77,9 +77,9 @@ func (s *testSplitIndex) TestgetStepValue(c *C) { } for _, ca := range cases { - l := longestCommonPrefixLen(ca.min, ca.max) + l := longestCommonPrefixLen(ca.lower, ca.upper) c.Assert(l, Equals, ca.l) - v0 := getStepValue(ca.min[l:], ca.max[l:], 1) + v0 := getStepValue(ca.lower[l:], ca.upper[l:], 1) c.Assert(v0, Equals, ca.v) } } @@ -126,8 +126,8 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { baseExecutor: newBaseExecutor(ctx, nil, nil), tableInfo: tbInfo, indexInfo: idxInfo, - min: []types.Datum{types.NewDatum(0)}, - max: []types.Datum{types.NewDatum(100)}, + lower: []types.Datum{types.NewDatum(0)}, + upper: []types.Datum{types.NewDatum(100)}, num: 10, } valueList, err := e.getSplitIdxKeys() @@ -179,8 +179,8 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // . // . // region26: y ~ +inf - e.min = []types.Datum{types.NewDatum("a")} - e.max = []types.Datum{types.NewDatum("z")} + e.lower = []types.Datum{types.NewDatum("a")} + e.upper = []types.Datum{types.NewDatum("z")} e.num = 26 // change index column type to varchar tbInfo.Columns[0].FieldType = *types.NewFieldType(mysql.TypeVarchar) @@ -227,16 +227,16 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // . // . // region10: 2019-01-01 00:00:00 ~ +inf - minTime := types.Time{ + lowerTime := types.Time{ Time: types.FromDate(2010, 1, 1, 0, 0, 0, 0), Type: mysql.TypeTimestamp, } - maxTime := types.Time{ + upperTime := types.Time{ Time: types.FromDate(2020, 1, 1, 0, 0, 0, 0), Type: mysql.TypeTimestamp, } - e.min = []types.Datum{types.NewDatum(minTime)} - e.max = []types.Datum{types.NewDatum(maxTime)} + e.lower = []types.Datum{types.NewDatum(lowerTime)} + e.upper = []types.Datum{types.NewDatum(upperTime)} e.num = 10 // change index column type to timestamp diff --git a/go.mod b/go.mod index 30d2fe922c113..5cfefd8cf1468 100644 --- a/go.mod +++ b/go.mod @@ -76,4 +76,4 @@ require ( sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190605023214-6d2c7a4ab128 +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190605052038-b1f406a4c07a diff --git a/go.sum b/go.sum index d50c7d7ffa9ac..7a705a5075539 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/crazycs520/parser v0.0.0-20190520125827-10bf17237e4a h1:AHsi/YPtUbo3q github.com/crazycs520/parser v0.0.0-20190520125827-10bf17237e4a/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/crazycs520/parser v0.0.0-20190605023214-6d2c7a4ab128 h1:QvTnMpYz8ERA5XiL88yDzoSrKUmudidxg/Sc+qUddR8= github.com/crazycs520/parser v0.0.0-20190605023214-6d2c7a4ab128/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= +github.com/crazycs520/parser v0.0.0-20190605052038-b1f406a4c07a h1:B60W9nl9orwDUn636cLZE/EiFbgcZ+7ltJV4UcQpuNw= +github.com/crazycs520/parser v0.0.0-20190605052038-b1f406a4c07a/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index e60b2e57ee5b8..5203776a52c52 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -485,8 +485,8 @@ type SplitRegion struct { TableInfo *model.TableInfo IndexInfo *model.IndexInfo - Min []types.Datum - Max []types.Datum + Lower []types.Datum + Upper []types.Datum Num int ValueLists [][]types.Datum } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index e1d50a5ad9425..6c3c7473137ca 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1687,8 +1687,8 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) return p, nil } - // Split index regions by min, max value. - checkMinMaxValue := func(valuesItem []ast.ExprNode, name string) ([]types.Datum, error) { + // Split index regions by lower, upper value. + checkLowerUpperValue := func(valuesItem []ast.ExprNode, name string) ([]types.Datum, error) { if len(valuesItem) == 0 { return nil, errors.Errorf("Split index region `%v` %s value count should more than 0", indexInfo.Name, name) } @@ -1697,16 +1697,16 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) } return convertValue2ColumnType(valuesItem) } - minValues, err := checkMinMaxValue(node.SplitOpt.Min, "min") + lowerValues, err := checkLowerUpperValue(node.SplitOpt.Lower, "lower") if err != nil { return nil, err } - maxValues, err := checkMinMaxValue(node.SplitOpt.Max, "max") + UpperValues, err := checkLowerUpperValue(node.SplitOpt.Upper, "upper") if err != nil { return nil, err } - p.Min = minValues - p.Max = maxValues + p.Lower = lowerValues + p.Upper = UpperValues if node.SplitOpt.Num > maxSplitRegionNum { return nil, errors.Errorf("Split index region num is exceed the limit %v", maxSplitRegionNum) From ffbe92287e5d09a6a6c211db89568fa5288b9bdf Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 5 Jun 2019 15:04:39 +0800 Subject: [PATCH 14/21] refine error msg --- executor/executor_test.go | 4 ++-- executor/split.go | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 900339192e5bb..7470ab7768fa0 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3804,9 +3804,9 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { // Check min value is more than max value. tk.MustExec(`split table t index idx1 between (0) and (1000000000) regions 10`) - _, err = tk.Exec(`split table t index idx1 between (2) and (1) regions 10`) + _, err = tk.Exec(`split table t index idx1 between (2,'a') and (1,'c') regions 10`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region `idx1` lower value [{1 0 0 0 2 [] }] should less than the upper value [{1 0 0 0 1 [] }]") + c.Assert(err.Error(), Equals, "Split index region `idx1` lower value (2,a) should less than the upper value (1,c)") // Check min value is invalid. _, err = tk.Exec(`split table t index idx1 between () and (1) regions 10`) diff --git a/executor/split.go b/executor/split.go index 09bd9a4672c4d..dc3aff5ef8380 100644 --- a/executor/split.go +++ b/executor/split.go @@ -120,11 +120,32 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { return nil, err } if bytes.Compare(lowerIdxKey, upperIdxKey) >= 0 { - return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper) + lowerStr, err1 := datumSliceToString(e.lower) + upperStr, err2 := datumSliceToString(e.upper) + if err1 != nil || err2 != nil { + return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper) + } + return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, lowerStr, upperStr) } return getValuesList(lowerIdxKey, upperIdxKey, e.num, idxKeys), nil } +func datumSliceToString(ds []types.Datum) (string, error) { + str := "(" + for i, d := range ds { + s, err := d.ToString() + if err != nil { + return str, err + } + if i > 0 { + str += "," + } + str += s + } + str += ")" + return str, nil +} + // longestCommonPrefixLen gets the longest common prefix byte length. func longestCommonPrefixLen(s1, s2 []byte) int { l := len(s1) From 289dbb3d7deaa2f8956d9314b86b0c1bfbe2078d Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 5 Jun 2019 15:06:14 +0800 Subject: [PATCH 15/21] refine code --- executor/split.go | 64 +++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/executor/split.go b/executor/split.go index dc3aff5ef8380..8948623749b0f 100644 --- a/executor/split.go +++ b/executor/split.go @@ -130,20 +130,26 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { return getValuesList(lowerIdxKey, upperIdxKey, e.num, idxKeys), nil } -func datumSliceToString(ds []types.Datum) (string, error) { - str := "(" - for i, d := range ds { - s, err := d.ToString() - if err != nil { - return str, err - } - if i > 0 { - str += "," - } - str += s +// getValuesList is used to get `num` values between lower and upper value. +// To Simplify the explain, suppose lower and upper value type is int64, and lower=0, upper=100, num=10, +// then calculate the step=(upper-lower)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. +// then the function will return [10,20,30,40,50,60,70,80,90]. +// The difference is the value type of upper,lower is []byte, So I use getUint64FromBytes to convert []byte to uint64. +func getValuesList(lower, upper []byte, num int, valuesList [][]byte) [][]byte { + commonPrefixIdx := longestCommonPrefixLen(lower, upper) + step := getStepValue(lower[commonPrefixIdx:], upper[commonPrefixIdx:], num) + startV := getUint64FromBytes(lower[commonPrefixIdx:], 0) + // To get `num` regions, only need to split `num-1` idx keys. + buf := make([]byte, 8) + for i := 0; i < num-1; i++ { + value := make([]byte, 0, commonPrefixIdx+8) + value = append(value, lower[:commonPrefixIdx]...) + startV += step + binary.BigEndian.PutUint64(buf, startV) + value = append(value, buf...) + valuesList = append(valuesList, value) } - str += ")" - return str, nil + return valuesList } // longestCommonPrefixLen gets the longest common prefix byte length. @@ -183,24 +189,18 @@ func getUint64FromBytes(bs []byte, pad byte) uint64 { return binary.BigEndian.Uint64(buf) } -// getValuesList is used to get `num` values between lower and upper value. -// To Simplify the explain, suppose lower and upper value type is int64, and lower=0, upper=100, num=10, -// then calculate the step=(upper-lower)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. -// then the function will return [10,20,30,40,50,60,70,80,90]. -// The difference is the value type of upper,lower is []byte, So I use getUint64FromBytes to convert []byte to uint64. -func getValuesList(lower, upper []byte, num int, valuesList [][]byte) [][]byte { - commonPrefixIdx := longestCommonPrefixLen(lower, upper) - step := getStepValue(lower[commonPrefixIdx:], upper[commonPrefixIdx:], num) - startV := getUint64FromBytes(lower[commonPrefixIdx:], 0) - // To get `num` regions, only need to split `num-1` idx keys. - buf := make([]byte, 8) - for i := 0; i < num-1; i++ { - value := make([]byte, 0, commonPrefixIdx+8) - value = append(value, lower[:commonPrefixIdx]...) - startV += step - binary.BigEndian.PutUint64(buf, startV) - value = append(value, buf...) - valuesList = append(valuesList, value) +func datumSliceToString(ds []types.Datum) (string, error) { + str := "(" + for i, d := range ds { + s, err := d.ToString() + if err != nil { + return str, err + } + if i > 0 { + str += "," + } + str += s } - return valuesList + str += ")" + return str, nil } From ab2c03d0502c7fde83a2fcd3440d2b19b1c326e4 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 5 Jun 2019 18:35:40 +0800 Subject: [PATCH 16/21] address comment --- executor/executor_test.go | 5 +++++ executor/split.go | 6 ++---- planner/core/planbuilder.go | 13 ++++++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 7470ab7768fa0..2198d1fdef19d 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3827,6 +3827,11 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 0`) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "Split index region num should more than 0") + + // Test truncate error msg. + _, err = tk.Exec(`split table t index idx1 between ("aa") and (1000000000) regions 0`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for index column 'b'") } func (s *testSuite) TestIssue10435(c *C) { diff --git a/executor/split.go b/executor/split.go index 8948623749b0f..a402408bc31ba 100644 --- a/executor/split.go +++ b/executor/split.go @@ -19,6 +19,7 @@ import ( "encoding/binary" "math" + "github.com/cznic/mathutil" "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" @@ -154,10 +155,7 @@ func getValuesList(lower, upper []byte, num int, valuesList [][]byte) [][]byte { // longestCommonPrefixLen gets the longest common prefix byte length. func longestCommonPrefixLen(s1, s2 []byte) int { - l := len(s1) - if len(s2) < len(s1) { - l = len(s2) - } + l := mathutil.Min(len(s1), len(s2)) i := 0 for ; i < l; i++ { if s1[i] != s2[i] { diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 6c3c7473137ca..f72ca401ce9ac 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1658,11 +1658,18 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) return nil, err } colOffset := indexInfo.Columns[j].Offset - value, err = value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) + value1, err := value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) if err != nil { - return nil, err + if !types.ErrTruncated.Equal(err) { + return nil, err + } + valStr, err1 := value.ToString() + if err1 != nil { + return nil, err + } + return nil, types.ErrTruncated.GenWithStack("Incorrect value: '%-.128s' for index column '%.192s'", valStr, tblInfo.Columns[colOffset].Name.O) } - values = append(values, value) + values = append(values, value1) } return values, nil } From ccb183be0e949140450c9efbe29f4f4a9cb42669 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 5 Jun 2019 18:37:41 +0800 Subject: [PATCH 17/21] update go.mod for parsre --- go.mod | 4 +--- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 5cfefd8cf1468..3a29cf4b42b3d 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190528074401-b942b3f4108f github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190604062806-1e93bd08fe82 + github.com/pingcap/parser v0.0.0-20190605085902-82be48fe0267 github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 @@ -75,5 +75,3 @@ require ( sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) - -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190605052038-b1f406a4c07a diff --git a/go.sum b/go.sum index 7a705a5075539..380316f3e39cb 100644 --- a/go.sum +++ b/go.sum @@ -30,12 +30,6 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/crazycs520/parser v0.0.0-20190520125827-10bf17237e4a h1:AHsi/YPtUbo3qHOGrpjxaUTroq1Hh31BaxM+LUIRFvQ= -github.com/crazycs520/parser v0.0.0-20190520125827-10bf17237e4a/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= -github.com/crazycs520/parser v0.0.0-20190605023214-6d2c7a4ab128 h1:QvTnMpYz8ERA5XiL88yDzoSrKUmudidxg/Sc+qUddR8= -github.com/crazycs520/parser v0.0.0-20190605023214-6d2c7a4ab128/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= -github.com/crazycs520/parser v0.0.0-20190605052038-b1f406a4c07a h1:B60W9nl9orwDUn636cLZE/EiFbgcZ+7ltJV4UcQpuNw= -github.com/crazycs520/parser v0.0.0-20190605052038-b1f406a4c07a/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= @@ -171,6 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190528074401-b942b3f4108f/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= +github.com/pingcap/parser v0.0.0-20190605085902-82be48fe0267 h1:LAR69Ocf7MIxHuh0uxr4JjRfPiFpKHq6aqEkvD63Z/k= +github.com/pingcap/parser v0.0.0-20190605085902-82be48fe0267/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From 5cdbbf18d251c5253fc35670b5255d8a02e37ce0 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 5 Jun 2019 18:46:33 +0800 Subject: [PATCH 18/21] fix ci --- planner/core/common_plans.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 5203776a52c52..3c3d8e034e058 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -479,7 +479,7 @@ type LoadStats struct { Path string } -// SplitIndexRegion represents a split index regions plan. +// SplitRegion represents a split regions plan. type SplitRegion struct { baseSchemaProducer From 95ec64b210e17b58d0978d17881bee8b6739efc0 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 6 Jun 2019 02:24:36 +0800 Subject: [PATCH 19/21] address comment --- executor/split.go | 6 +++--- executor/split_test.go | 34 +++++++++++++++++----------------- planner/core/planbuilder.go | 4 ++-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/executor/split.go b/executor/split.go index a402408bc31ba..23d1f3cc0afda 100644 --- a/executor/split.go +++ b/executor/split.go @@ -116,7 +116,7 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { if err != nil { return nil, err } - upperIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.upper, math.MinInt64, nil) + upperIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.upper, math.MaxInt64, nil) if err != nil { return nil, err } @@ -134,7 +134,7 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { // getValuesList is used to get `num` values between lower and upper value. // To Simplify the explain, suppose lower and upper value type is int64, and lower=0, upper=100, num=10, // then calculate the step=(upper-lower)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values. -// then the function will return [10,20,30,40,50,60,70,80,90]. +// Then the function will return [10,20,30,40,50,60,70,80,90]. // The difference is the value type of upper,lower is []byte, So I use getUint64FromBytes to convert []byte to uint64. func getValuesList(lower, upper []byte, num int, valuesList [][]byte) [][]byte { commonPrefixIdx := longestCommonPrefixLen(lower, upper) @@ -166,7 +166,7 @@ func longestCommonPrefixLen(s1, s2 []byte) int { } // getStepValue gets the step of between the lower and upper value. step = (upper-lower)/num. -// convert byte slice to uint64 first. +// Convert byte slice to uint64 first. func getStepValue(lower, upper []byte, num int) uint64 { lowerUint := getUint64FromBytes(lower, 0) upperUint := getUint64FromBytes(upper, 0xff) diff --git a/executor/split_test.go b/executor/split_test.go index da4545088a08b..c73d37a5d65dd 100644 --- a/executor/split_test.go +++ b/executor/split_test.go @@ -110,17 +110,17 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // Test for int index. // range is 0 ~ 100, and split into 10 region. - // So 10 regions range is like below: - // region1: -inf ~ 10 - // region2: 10 ~ 20 - // region3: 20 ~ 30 - // region4: 30 ~ 40 - // region5: 40 ~ 50 - // region6: 50 ~ 60 - // region7: 60 ~ 70 - // region8: 70 ~ 80 - // region9: 80 ~ 80 - // region10: 90 ~ +inf + // So 10 regions range is like below, left close right open interval: + // region1: [-inf ~ 10) + // region2: [10 ~ 20) + // region3: [20 ~ 30) + // region4: [30 ~ 40) + // region5: [40 ~ 50) + // region6: [50 ~ 60) + // region7: [60 ~ 70) + // region8: [70 ~ 80) + // region9: [80 ~ 90) + // region10: [90 ~ +inf) ctx := mock.NewContext() e := &SplitIndexRegionExec{ baseExecutor: newBaseExecutor(ctx, nil, nil), @@ -173,12 +173,12 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // Test for varchar index. // range is a ~ z, and split into 26 region. // So 26 regions range is like below: - // region1: -inf ~ b - // region2: b ~ c + // region1: [-inf ~ b) + // region2: [b ~ c) // . // . // . - // region26: y ~ +inf + // region26: [y ~ +inf) e.lower = []types.Datum{types.NewDatum("a")} e.upper = []types.Datum{types.NewDatum("z")} e.num = 26 @@ -221,12 +221,12 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { // Test for timestamp index. // range is 2010-01-01 00:00:00 ~ 2020-01-01 00:00:00, and split into 10 region. // So 10 regions range is like below: - // region1: -inf ~ 2011-01-01 00:00:00 - // region2: 2011-01-01 00:00:00 ~ 2012-01-01 00:00:00 + // region1: [-inf ~ 2011-01-01 00:00:00) + // region2: [2011-01-01 00:00:00 ~ 2012-01-01 00:00:00) // . // . // . - // region10: 2019-01-01 00:00:00 ~ +inf + // region10: [2019-01-01 00:00:00 ~ +inf) lowerTime := types.Time{ Time: types.FromDate(2010, 1, 1, 0, 0, 0, 0), Type: mysql.TypeTimestamp, diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index f72ca401ce9ac..13f89fc881c8b 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1708,12 +1708,12 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) if err != nil { return nil, err } - UpperValues, err := checkLowerUpperValue(node.SplitOpt.Upper, "upper") + upperValues, err := checkLowerUpperValue(node.SplitOpt.Upper, "upper") if err != nil { return nil, err } p.Lower = lowerValues - p.Upper = UpperValues + p.Upper = upperValues if node.SplitOpt.Num > maxSplitRegionNum { return nil, errors.Errorf("Split index region num is exceed the limit %v", maxSplitRegionNum) From a2037dda3f752f78d3913259f78db12fbf3be470 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 6 Jun 2019 02:43:21 +0800 Subject: [PATCH 20/21] add comment and fix test --- executor/split.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/executor/split.go b/executor/split.go index 23d1f3cc0afda..255caaf94b118 100644 --- a/executor/split.go +++ b/executor/split.go @@ -116,7 +116,9 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { if err != nil { return nil, err } - upperIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.upper, math.MaxInt64, nil) + // Use math.MinInt64 as handle_id for the upper index key to avoid affecting calculate split point. + // If use math.MaxInt64 here, test of `TestSplitIndex` will report error. + upperIdxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, e.upper, math.MinInt64, nil) if err != nil { return nil, err } From e5ba5b65c327f86fe1f8decfe7a86f226c691d51 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 6 Jun 2019 10:37:37 +0800 Subject: [PATCH 21/21] address comment --- planner/core/planbuilder.go | 87 +++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 13f89fc881c8b..117aa12a80c01 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1632,47 +1632,6 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo) mockTablePlan.SetSchema(schema) - convertValue2ColumnType := func(valuesItem []ast.ExprNode) ([]types.Datum, error) { - values := make([]types.Datum, 0, len(valuesItem)) - for j, valueItem := range valuesItem { - var expr expression.Expression - var err error - switch x := valueItem.(type) { - case *driver.ValueExpr: - expr = &expression.Constant{ - Value: x.Datum, - RetType: &x.Type, - } - default: - expr, _, err = b.rewrite(valueItem, mockTablePlan, nil, true) - if err != nil { - return nil, err - } - } - constant, ok := expr.(*expression.Constant) - if !ok { - return nil, errors.New("expect constant values") - } - value, err := constant.Eval(chunk.Row{}) - if err != nil { - return nil, err - } - colOffset := indexInfo.Columns[j].Offset - value1, err := value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) - if err != nil { - if !types.ErrTruncated.Equal(err) { - return nil, err - } - valStr, err1 := value.ToString() - if err1 != nil { - return nil, err - } - return nil, types.ErrTruncated.GenWithStack("Incorrect value: '%-.128s' for index column '%.192s'", valStr, tblInfo.Columns[colOffset].Name.O) - } - values = append(values, value1) - } - return values, nil - } p := &SplitRegion{ TableInfo: tblInfo, IndexInfo: indexInfo, @@ -1684,7 +1643,7 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) if len(valuesItem) > len(indexInfo.Columns) { return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) } - values, err := convertValue2ColumnType(valuesItem) + values, err := b.convertValue2ColumnType(valuesItem, mockTablePlan, indexInfo, tblInfo) if err != nil { return nil, err } @@ -1702,7 +1661,7 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) if len(valuesItem) > len(indexInfo.Columns) { return nil, errors.Errorf("Split index region `%v` Column count doesn't match value count at %v", indexInfo.Name, name) } - return convertValue2ColumnType(valuesItem) + return b.convertValue2ColumnType(valuesItem, mockTablePlan, indexInfo, tblInfo) } lowerValues, err := checkLowerUpperValue(node.SplitOpt.Lower, "lower") if err != nil { @@ -1724,6 +1683,48 @@ func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) return p, nil } +func (b *PlanBuilder) convertValue2ColumnType(valuesItem []ast.ExprNode, mockTablePlan LogicalPlan, indexInfo *model.IndexInfo, tblInfo *model.TableInfo) ([]types.Datum, error) { + values := make([]types.Datum, 0, len(valuesItem)) + for j, valueItem := range valuesItem { + var expr expression.Expression + var err error + switch x := valueItem.(type) { + case *driver.ValueExpr: + expr = &expression.Constant{ + Value: x.Datum, + RetType: &x.Type, + } + default: + expr, _, err = b.rewrite(valueItem, mockTablePlan, nil, true) + if err != nil { + return nil, err + } + } + constant, ok := expr.(*expression.Constant) + if !ok { + return nil, errors.New("expect constant values") + } + value, err := constant.Eval(chunk.Row{}) + if err != nil { + return nil, err + } + colOffset := indexInfo.Columns[j].Offset + value1, err := value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) + if err != nil { + if !types.ErrTruncated.Equal(err) { + return nil, err + } + valStr, err1 := value.ToString() + if err1 != nil { + return nil, err + } + return nil, types.ErrTruncated.GenWithStack("Incorrect value: '%-.128s' for index column '%.192s'", valStr, tblInfo.Columns[colOffset].Name.O) + } + values = append(values, value1) + } + return values, nil +} + func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) { var authErr error switch v := node.(type) {