Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: add split index region syntax #10203

Merged
merged 30 commits into from
May 6, 2019
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
5c613ad
init
crazycs520 Apr 10, 2019
2bc50c8
add scatter
crazycs520 Apr 12, 2019
a88ea9b
fix test
crazycs520 Apr 12, 2019
3145795
only split region for positive num
crazycs520 Apr 12, 2019
2fccfab
add PreSplitRegions option
crazycs520 Apr 12, 2019
77f0a96
add retry for scatter region
crazycs520 Apr 12, 2019
189335d
refine comment
crazycs520 Apr 12, 2019
03783c0
Merge branch 'master' into splite-region
crazycs520 Apr 12, 2019
b517f03
remove blank line
crazycs520 Apr 15, 2019
f96d8d3
Merge branch 'master' into splite-region
crazycs520 Apr 16, 2019
55cd7be
update go.mod for pd
crazycs520 Apr 16, 2019
eb27dfb
add wait scatter finish
crazycs520 Apr 17, 2019
a70696f
add table option: wait_split_finish
crazycs520 Apr 17, 2019
df45ac4
Merge branch 'master' of https://github.com/pingcap/tidb into splite-…
crazycs520 Apr 18, 2019
e362d68
split and scatter region first, wait scatter finish together
crazycs520 Apr 18, 2019
918b183
*: add split table index syntax
crazycs520 Apr 19, 2019
87ec41a
wait scatter finish when create table
crazycs520 Apr 22, 2019
58c4a9e
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 Apr 25, 2019
6cfd51e
refine code
crazycs520 Apr 25, 2019
f1b22a3
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 Apr 25, 2019
952b870
add comment
crazycs520 Apr 25, 2019
38a057d
address comment and add test for convert value should not ignore trun…
crazycs520 Apr 26, 2019
53ccd4f
Merge branch 'master' into split_index
crazycs520 Apr 26, 2019
b44db0b
use session variable tidb_wait_table_split_finish to chose sync or as…
crazycs520 Apr 28, 2019
15c9514
update go mod
crazycs520 Apr 29, 2019
650c428
address comment
crazycs520 May 6, 2019
77ca0d1
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 May 6, 2019
1d2e9c9
address comment
crazycs520 May 6, 2019
44016a5
address comment and update go.mod
crazycs520 May 6, 2019
652e95c
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 May 6, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions executor/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor {
return b.buildWindow(v)
case *plannercore.SQLBindPlan:
return b.buildSQLBindExec(v)
case *plannercore.SplitIndexRegion:
return b.buildSplitIndexRegion(v)
default:
if mp, ok := p.(MockPhysicalPlan); ok {
return mp.GetExecutor()
Expand Down Expand Up @@ -1249,6 +1251,17 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo
return e
}

func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitIndexRegion) Executor {
base := newBaseExecutor(b.ctx, nil, v.ExplainID())
base.initCap = chunk.ZeroCapacity
return &SplitIndexRegionExec{
baseExecutor: base,
table: v.Table,
indexInfo: v.IndexInfo,
valueLists: v.ValueLists,
}
}

func (b *executorBuilder) buildUpdate(v *plannercore.Update) Executor {
tblID2table := make(map[int64]table.Table)
for id := range v.SelectPlan.Schema().TblID2Handle {
Expand Down
78 changes: 78 additions & 0 deletions executor/split.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package executor

import (
"context"

"github.com/pingcap/parser/model"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
)

// SplitIndexRegionExec represents a split index regions executor.
type SplitIndexRegionExec struct {
baseExecutor

table table.Table
indexInfo *model.IndexInfo
valueLists [][]types.Datum
}

type splitableStore interface {
SplitRegionAndScatter(splitKey kv.Key) (uint64, error)
WaitScatterRegionFinish(regionID uint64) error
}

// Next implements the Executor Next interface.
func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) error {
store := e.ctx.GetStore()
s, ok := store.(splitableStore)
if !ok {
zimulala marked this conversation as resolved.
Show resolved Hide resolved
return nil
}
regionIDs := make([]uint64, 0, len(e.valueLists))
index := tables.NewIndex(e.table.Meta().ID, e.table.Meta(), e.indexInfo)
for _, values := range e.valueLists {
idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, 1, nil)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1 means min handle id?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, Maybe 0 is better?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can use a constant, better than hard code!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@winkyao
I think 1 isn't always the min handle. If a table has a PKIsHandle column, then the handle may be a negative value.
So I think we need to use math.MinInt64.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@zimulala The key is encoding in binary format, so negative value is always greater than a positive value. @nolouch Right?

Copy link
Contributor Author

@crazycs520 crazycs520 May 6, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is a test:
0 encode to byte is: [128 0 0 0 0 0 0 0]
math.MinInt64 encode to byte is: [0 0 0 0 0 0 0 0]

so, math.MinInt64 is better.

if err != nil {
return err
}

regionID, err := s.SplitRegionAndScatter(idxKey)
if err != nil {
logutil.Logger(context.Background()).Warn("split table index region failed",
zap.String("table", e.table.Meta().Name.L),
zap.String("index", e.indexInfo.Name.L),
zap.Error(err))
continue
}
regionIDs = append(regionIDs, regionID)

}
for _, regionID := range regionIDs {
err := s.WaitScatterRegionFinish(regionID)
if err != nil {
logutil.Logger(context.Background()).Warn("wait scatter region failed",
zap.Uint64("regionID", regionID),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to add the table and index information to this log?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

zap.Error(err))
}
}
return nil
}
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,5 @@ require (
sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
)

replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190424150639-ab99f9af0665
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/crazycs520/parser v0.0.0-20190424150639-ab99f9af0665 h1:eFA/z4xCptmSRH9b3N2NnKtdr5ShMWLSoXlClAv+MCA=
github.com/crazycs520/parser v0.0.0-20190424150639-ab99f9af0665/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE=
Expand Down Expand Up @@ -160,8 +162,6 @@ github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXC
github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w=
github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ=
github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw=
github.com/pingcap/parser v0.0.0-20190424024541-e2cdb851bce2 h1:WZPcqMEpW1PFsusiCEiNlatZdI8dGURXneIUrFNmehI=
github.com/pingcap/parser v0.0.0-20190424024541-e2cdb851bce2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4=
github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM=
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU=
Expand Down
10 changes: 10 additions & 0 deletions planner/core/common_plans.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/types"
driver "github.com/pingcap/tidb/types/parser_driver"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/kvcache"
Expand Down Expand Up @@ -475,6 +476,15 @@ type LoadStats struct {
Path string
}

// SplitIndexRegion represents a split index regions plan.
type SplitIndexRegion struct {
baseSchemaProducer

Table table.Table
IndexInfo *model.IndexInfo
ValueLists [][]types.Datum
}

// DDL represents a DDL statement plan.
type DDL struct {
baseSchemaProducer
Expand Down
50 changes: 50 additions & 0 deletions planner/core/planbuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,8 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) {
return b.buildDropBindPlan(x)
case *ast.ChangeStmt:
return b.buildChange(x)
case *ast.SplitIndexRegionStmt:
return b.buildSplitIndexRegion(x)
}
return nil, ErrUnsupportedType.GenWithStack("Unsupported type %T", node)
}
Expand Down Expand Up @@ -1588,6 +1590,54 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan {
return p
}

func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Plan, error) {
tblInfo := node.Table.TableInfo
indexInfo := tblInfo.FindIndexByName(strings.ToLower(node.IndexName))
if indexInfo == nil {
return nil, ErrKeyDoesNotExist.GenWithStackByArgs(node.IndexName, tblInfo.Name)
}
schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo)
mockTablePlan := LogicalTableDual{}.Init(b.ctx)
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
mockTablePlan.SetSchema(schema)

tps := make([]*types.FieldType, 0, len(indexInfo.Columns))
for _, c := range indexInfo.Columns {
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
col := tblInfo.Columns[c.Offset]
tps = append(tps, &col.FieldType)
}

indexValues := make([][]types.Datum, 0, len(node.ValueLists))
for i, valuesItem := range node.ValueLists {
if len(valuesItem) > len(indexInfo.Columns) {
return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1)
}
valueList := make([]types.Datum, 0, len(valuesItem))
for j, valueItem := range valuesItem {
x, ok := valueItem.(*driver.ValueExpr)
if !ok {
return nil, errors.Trace(errors.New("expect constant values"))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we remove this trace? Other places don't use it.
And we can print the value to the error message.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

}
value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, tps[j])
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, err
}

valueList = append(valueList, value)
}
indexValues = append(indexValues, valueList)
}
tableInPlan, ok := b.is.TableByID(tblInfo.ID)
if !ok {
return nil, errors.Errorf("Can't get table %s.", tblInfo.Name.O)
}
return &SplitIndexRegion{
Table: tableInPlan,
IndexInfo: indexInfo,
ValueLists: indexValues,
}, nil

}

func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) {
var authErr error
switch v := node.(type) {
Expand Down
1 change: 0 additions & 1 deletion store/tikv/split_region.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package tikv
import (
"bytes"
"context"

crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/kvproto/pkg/metapb"
Expand Down