From ea6b1de3d00f6821db1897512608bcdbe3305f93 Mon Sep 17 00:00:00 2001 From: lzmhhh123 Date: Fri, 8 Nov 2019 16:23:04 +0800 Subject: [PATCH 1/4] fix invalid range for tiflash --- store/tikv/coprocessor.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index 21482a2c0708f..3897da134109a 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -17,6 +17,9 @@ import ( "bytes" "context" "fmt" + "github.com/pingcap/tidb/distsql" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/ranger" "io" "sort" "strings" @@ -218,6 +221,13 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, req *kv if req.Streaming { cmdType = tikvrpc.CmdCopStream } + var tableStart, tableEnd kv.Key + if req.StoreType == kv.TiFlash { + tableID := tablecodec.DecodeTableID(ranges.at(0).StartKey) + fullRange := ranger.FullIntRange(false) + keyRange := distsql.TableRangesToKVRanges(tableID, fullRange, nil) + tableStart, tableEnd = keyRange[0].StartKey, keyRange[0].EndKey + } var tasks []*copTask appendTask := func(regionWithRangeInfo *KeyLocation, ranges *copRanges) { @@ -239,7 +249,14 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, req *kv i = nextI } } else if req.StoreType == kv.TiFlash { - fullRange := kv.KeyRange{StartKey: regionWithRangeInfo.StartKey, EndKey: regionWithRangeInfo.EndKey} + left, right := regionWithRangeInfo.StartKey, regionWithRangeInfo.EndKey + if bytes.Compare(tableStart, left) >= 0 || len(left) == 0{ + left = tableStart + } + if bytes.Compare(tableEnd, right) <= 0 || len(right) == 0 { + right = tableEnd + } + fullRange := kv.KeyRange{StartKey: left, EndKey: right} tasks = append(tasks, &copTask{ region: regionWithRangeInfo.Region, // TiFlash only support full range scan for the region, ignore the real ranges From 55f850c1da614bd72eab37dc7fe2f0133ae26040 Mon Sep 17 00:00:00 2001 From: lzmhhh123 Date: Fri, 8 Nov 2019 16:26:06 +0800 Subject: [PATCH 2/4] fix format --- store/tikv/coprocessor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index 3897da134109a..51118d94dbddf 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -17,9 +17,6 @@ import ( "bytes" "context" "fmt" - "github.com/pingcap/tidb/distsql" - "github.com/pingcap/tidb/tablecodec" - "github.com/pingcap/tidb/util/ranger" "io" "sort" "strings" @@ -33,12 +30,15 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" + "github.com/pingcap/tidb/util/ranger" "go.uber.org/zap" ) @@ -250,7 +250,7 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, req *kv } } else if req.StoreType == kv.TiFlash { left, right := regionWithRangeInfo.StartKey, regionWithRangeInfo.EndKey - if bytes.Compare(tableStart, left) >= 0 || len(left) == 0{ + if bytes.Compare(tableStart, left) >= 0 || len(left) == 0 { left = tableStart } if bytes.Compare(tableEnd, right) <= 0 || len(right) == 0 { From d0a5c54998b754a54d16163a7243aa7e451e8ded Mon Sep 17 00:00:00 2001 From: lzmhhh123 Date: Fri, 8 Nov 2019 17:06:46 +0800 Subject: [PATCH 3/4] fix ci --- store/tikv/coprocessor_test.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/store/tikv/coprocessor_test.go b/store/tikv/coprocessor_test.go index 63e49393c736a..3e25f6e26e458 100644 --- a/store/tikv/coprocessor_test.go +++ b/store/tikv/coprocessor_test.go @@ -50,7 +50,7 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "c"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 1) - s.taskEqual(c, tasks[0], regionIDs[0], "", "g") + s.taskEqual(c, tasks[0], regionIDs[0], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "g") tasks, err = buildCopTasks(bo, cache, buildCopRanges("g", "n"), req) c.Assert(err, IsNil) @@ -60,7 +60,7 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("g", "n"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 1) - s.taskEqual(c, tasks[0], regionIDs[1], "g", "n") + s.taskEqual(c, tasks[0], regionIDs[1], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "n") tasks, err = buildCopTasks(bo, cache, buildCopRanges("m", "n"), req) c.Assert(err, IsNil) @@ -70,7 +70,7 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("m", "n"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 1) - s.taskEqual(c, tasks[0], regionIDs[1], "g", "n") + s.taskEqual(c, tasks[0], regionIDs[1], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "n") tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "k"), req) c.Assert(err, IsNil) @@ -81,8 +81,8 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "k"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 2) - s.taskEqual(c, tasks[0], regionIDs[0], "", "g") - s.taskEqual(c, tasks[1], regionIDs[1], "g", "n") + s.taskEqual(c, tasks[0], regionIDs[0], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "g") + s.taskEqual(c, tasks[1], regionIDs[1], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "n") tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "x"), req) c.Assert(err, IsNil) @@ -95,10 +95,10 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "x"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 4) - s.taskEqual(c, tasks[0], regionIDs[0], "", "g") - s.taskEqual(c, tasks[1], regionIDs[1], "g", "n") - s.taskEqual(c, tasks[2], regionIDs[2], "n", "t") - s.taskEqual(c, tasks[3], regionIDs[3], "t", "") + s.taskEqual(c, tasks[0], regionIDs[0], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "g") + s.taskEqual(c, tasks[1], regionIDs[1], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "n") + s.taskEqual(c, tasks[2], regionIDs[2], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "t") + s.taskEqual(c, tasks[3], regionIDs[3], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "t\x80\x00\x00\x00\x00\x00\x00\x00_r\xff\xff\xff\xff\xff\xff\xff\xff\x00") tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "b", "b", "c"), req) c.Assert(err, IsNil) @@ -108,7 +108,7 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "b", "b", "c"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 1) - s.taskEqual(c, tasks[0], regionIDs[0], "", "g") + s.taskEqual(c, tasks[0], regionIDs[0], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "g") tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "b", "e", "f"), req) c.Assert(err, IsNil) @@ -118,7 +118,7 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("a", "b", "e", "f"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 1) - s.taskEqual(c, tasks[0], regionIDs[0], "", "g") + s.taskEqual(c, tasks[0], regionIDs[0], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "g") tasks, err = buildCopTasks(bo, cache, buildCopRanges("g", "n", "o", "p"), req) c.Assert(err, IsNil) @@ -129,8 +129,8 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("g", "n", "o", "p"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 2) - s.taskEqual(c, tasks[0], regionIDs[1], "g", "n") - s.taskEqual(c, tasks[1], regionIDs[2], "n", "t") + s.taskEqual(c, tasks[0], regionIDs[1], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "n") + s.taskEqual(c, tasks[1], regionIDs[2], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "t") tasks, err = buildCopTasks(bo, cache, buildCopRanges("h", "k", "m", "p"), req) c.Assert(err, IsNil) @@ -141,8 +141,8 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { tasks, err = buildCopTasks(bo, cache, buildCopRanges("h", "k", "m", "p"), flashReq) c.Assert(err, IsNil) c.Assert(tasks, HasLen, 2) - s.taskEqual(c, tasks[0], regionIDs[1], "g", "n") - s.taskEqual(c, tasks[1], regionIDs[2], "n", "t") + s.taskEqual(c, tasks[0], regionIDs[1], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "n") + s.taskEqual(c, tasks[1], regionIDs[2], "t\x80\x00\x00\x00\x00\x00\x00\x00_r\x00\x00\x00\x00\x00\x00\x00\x00", "t") } func (s *testCoprocessorSuite) TestSplitRegionRanges(c *C) { From 2ba0b2ccf824e3e3dfae10fea15e4b72c6d32786 Mon Sep 17 00:00:00 2001 From: lzmhhh123 Date: Fri, 8 Nov 2019 17:24:56 +0800 Subject: [PATCH 4/4] address comment --- store/tikv/coprocessor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index 51118d94dbddf..f2d9cfe93287c 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -250,7 +250,7 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, req *kv } } else if req.StoreType == kv.TiFlash { left, right := regionWithRangeInfo.StartKey, regionWithRangeInfo.EndKey - if bytes.Compare(tableStart, left) >= 0 || len(left) == 0 { + if bytes.Compare(tableStart, left) >= 0 { left = tableStart } if bytes.Compare(tableEnd, right) <= 0 || len(right) == 0 {