From 697138811880a85329fc4e684545d04e4ff4fcd1 Mon Sep 17 00:00:00 2001 From: silver__bullet Date: Sun, 17 Nov 2019 23:14:44 +0800 Subject: [PATCH 01/14] expression: implement vectorized evaluation for builtinGEIntSig (#13517) --- expression/builtin_compare_vec.go | 39 ++++++++++++++++++++++++-- expression/builtin_compare_vec_test.go | 21 ++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/expression/builtin_compare_vec.go b/expression/builtin_compare_vec.go index 074a4cad04bf4..a5abf3a0df021 100644 --- a/expression/builtin_compare_vec.go +++ b/expression/builtin_compare_vec.go @@ -164,11 +164,35 @@ func (b *builtinGreatestIntSig) vectorized() bool { } func (b *builtinGEIntSig) vectorized() bool { - return false + return true } func (b *builtinGEIntSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + var err error + var buf0, buf1 *chunk.Column + buf0, err = b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err = b.args[0].VecEvalInt(b.ctx, input, buf0); err != nil { + return err + } + buf1, err = b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf1) + if err = b.args[1].VecEvalInt(b.ctx, input, buf1); err != nil { + return err + } + + result.ResizeInt64(n, false) + vecCompareInt(mysql.HasUnsignedFlag(b.args[0].GetType().Flag), mysql.HasUnsignedFlag(b.args[1].GetType().Flag), buf0, buf1, result) + result.MergeNulls(buf0, buf1) + vecResOfGE(result.Int64s()) + return nil } func (b *builtinLeastRealSig) vectorized() bool { @@ -540,6 +564,17 @@ func vecResOfGT(res []int64) { } } +func vecResOfGE(res []int64) { + n := len(res) + for i := 0; i < n; i++ { + if res[i] >= 0 { + res[i] = 1 + } else { + res[i] = 0 + } + } +} + //vecCompareInt is vectorized CompareInt() func vecCompareInt(isUnsigned0, isUnsigned1 bool, largs, rargs, result *chunk.Column) { switch { diff --git a/expression/builtin_compare_vec_test.go b/expression/builtin_compare_vec_test.go index fd6d7d83dda75..a9480dcc4cdc2 100644 --- a/expression/builtin_compare_vec_test.go +++ b/expression/builtin_compare_vec_test.go @@ -98,8 +98,25 @@ var vecBuiltinCompareCases = map[string][]vecExprBenchCase{ }, }, }, - ast.EQ: {}, - ast.GE: {}, + ast.EQ: {}, + ast.GE: { + {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETInt, types.ETInt}}, + {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETInt, types.ETInt}, + childrenFieldTypes: []*types.FieldType{{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, + {Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, + }, + }, + {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETInt, types.ETInt}, + childrenFieldTypes: []*types.FieldType{{Tp: mysql.TypeLonglong}, + {Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, + }, + }, + {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETInt, types.ETInt}, + childrenFieldTypes: []*types.FieldType{{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, + {Tp: mysql.TypeLonglong}, + }, + }, + }, ast.Date: {}, ast.Greatest: { {retEvalType: types.ETDecimal, childrenTypes: []types.EvalType{types.ETDecimal, types.ETDecimal, types.ETDecimal}}, From aa8f6dfd60a749f7fa2b2f85f35963043ee65d60 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 18 Nov 2019 10:37:14 +0800 Subject: [PATCH 02/14] optimize dll test (#13497) --- ddl/db_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index 7b5e3b84bdc9a..d6ce71321ed43 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -1029,9 +1029,13 @@ LOOP: return } - // test index range - for i := 0; i < 100; i++ { - index := rand.Intn(len(keys) - 3) + // Test index range with lower/upper boundary and random inner cases + step := len(keys) / 20 + for i := 0; i <= 20; i++ { + index := i * step + if index > len(keys)-3 { + index = len(keys) - 3 + } rows := tk.MustQuery("select c1 from test_add_index where c3 >= ? order by c1 limit 3", keys[index]).Rows() matchRows(c, rows, [][]interface{}{{keys[index]}, {keys[index+1]}, {keys[index+2]}}) } From 0cc3dad864ae4c3fd2c46f8b488165c9a1a4200f Mon Sep 17 00:00:00 2001 From: mmyj <21154264+mmyj@users.noreply.github.com> Date: Mon, 18 Nov 2019 11:43:07 +0800 Subject: [PATCH 03/14] expression: implement vectorized evaluation for `builtinLpadBinarySig` (#13442) --- expression/builtin_string_vec.go | 65 ++++++++++++++++++++++++++- expression/builtin_string_vec_test.go | 12 +++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/expression/builtin_string_vec.go b/expression/builtin_string_vec.go index 6d0433c6e5698..a4ba894f7a258 100644 --- a/expression/builtin_string_vec.go +++ b/expression/builtin_string_vec.go @@ -739,11 +739,72 @@ func (b *builtinASCIISig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) e } func (b *builtinLpadBinarySig) vectorized() bool { - return false + return true } +// vecEvalString evals LPAD(str,len,padstr). +// See https://dev.mysql.com/doc/refman/5.7/en/string-functions.html#function_lpad func (b *builtinLpadBinarySig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + strBuf, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(strBuf) + if err := b.args[0].VecEvalString(b.ctx, input, strBuf); err != nil { + return err + } + lenBuf, err := b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + defer b.bufAllocator.put(lenBuf) + if err := b.args[1].VecEvalInt(b.ctx, input, lenBuf); err != nil { + return err + } + padBuf, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(padBuf) + if err := b.args[2].VecEvalString(b.ctx, input, padBuf); err != nil { + return err + } + + result.ReserveString(n) + i64s := lenBuf.Int64s() + lenBuf.MergeNulls(strBuf) + for i := 0; i < n; i++ { + if lenBuf.IsNull(i) { + result.AppendNull() + continue + } + targetLength := int(i64s[i]) + if uint64(targetLength) > b.maxAllowedPacket { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errWarnAllowedPacketOverflowed.GenWithStackByArgs("lpad", b.maxAllowedPacket)) + result.AppendNull() + continue + } + + if padBuf.IsNull(i) { + result.AppendNull() + continue + } + str := strBuf.GetString(i) + strLength := len(str) + padStr := padBuf.GetString(i) + padLength := len(padStr) + if targetLength < 0 || targetLength > b.tp.Flen || (strLength < targetLength && padLength == 0) { + result.AppendNull() + continue + } + if tailLen := targetLength - strLength; tailLen > 0 { + repeatCount := tailLen/padLength + 1 + str = strings.Repeat(padStr, repeatCount)[:tailLen] + str + } + result.AppendString(str[:targetLength]) + } + return nil } func (b *builtinLpadSig) vectorized() bool { diff --git a/expression/builtin_string_vec_test.go b/expression/builtin_string_vec_test.go index 17934ffcecddb..6a0e930fc62ff 100644 --- a/expression/builtin_string_vec_test.go +++ b/expression/builtin_string_vec_test.go @@ -135,6 +135,18 @@ var vecBuiltinStringCases = map[string][]vecExprBenchCase{ childrenTypes: []types.EvalType{types.ETString, types.ETInt, types.ETString}, geners: []dataGenerator{&defaultGener{0.2, types.ETString}, &defaultGener{0.2, types.ETInt}, &defaultGener{0.2, types.ETString}}, }, + { + retEvalType: types.ETString, + childrenTypes: []types.EvalType{types.ETString, types.ETInt, types.ETString}, + childrenFieldTypes: []*types.FieldType{{Tp: mysql.TypeString, Flag: mysql.BinaryFlag, Collate: charset.CollationBin}}, + geners: []dataGenerator{&randLenStrGener{0, 20}, &rangeInt64Gener{168435456, 368435456}, &randLenStrGener{0, 10}}, + }, + { + retEvalType: types.ETString, + childrenTypes: []types.EvalType{types.ETString, types.ETInt, types.ETString}, + childrenFieldTypes: []*types.FieldType{{Tp: mysql.TypeString, Flag: mysql.BinaryFlag, Collate: charset.CollationBin}}, + geners: []dataGenerator{&defaultGener{0.2, types.ETString}, &defaultGener{0.2, types.ETInt}, &defaultGener{0.2, types.ETString}}, + }, }, ast.Rpad: { { From a8e9fa3857ba07bd344f5bf8c381b923996d03ea Mon Sep 17 00:00:00 2001 From: Lanearth <949930437@qq.com> Date: Mon, 18 Nov 2019 12:04:54 +0800 Subject: [PATCH 04/14] expression: implement vectorized evaluation for `builtinWeekWithModeSig` (#13402) --- expression/builtin_time_vec.go | 48 +++++++++++++++++++++++++++-- expression/builtin_time_vec_test.go | 3 ++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/expression/builtin_time_vec.go b/expression/builtin_time_vec.go index 16e90cb3de2e6..2a15380cc760c 100644 --- a/expression/builtin_time_vec.go +++ b/expression/builtin_time_vec.go @@ -911,11 +911,55 @@ func (b *builtinQuarterSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) } func (b *builtinWeekWithModeSig) vectorized() bool { - return false + return true } func (b *builtinWeekWithModeSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + buf1, err := b.bufAllocator.get(types.ETDatetime, n) + if err != nil { + return err + } + if err := b.args[0].VecEvalTime(b.ctx, input, buf1); err != nil { + return err + } + defer b.bufAllocator.put(buf1) + + buf2, err := b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + if err := b.args[1].VecEvalInt(b.ctx, input, buf2); err != nil { + return err + } + defer b.bufAllocator.put(buf2) + + result.ResizeInt64(n, false) + i64s := result.Int64s() + ds := buf1.Times() + ms := buf2.Int64s() + for i := 0; i < n; i++ { + if buf1.IsNull(i) { + result.SetNull(i, true) + continue + } + date := ds[i] + if date.IsZero() { + if err := handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenWithStackByArgs(date.String())); err != nil { + return err + } + result.SetNull(i, true) + continue + } + if buf2.IsNull(i) { + result.SetNull(i, true) + continue + } + mode := int(ms[i]) + week := date.Time.Week(int(mode)) + i64s[i] = int64(week) + } + return nil } func (b *builtinExtractDurationSig) vectorized() bool { diff --git a/expression/builtin_time_vec_test.go b/expression/builtin_time_vec_test.go index 3b6bb3f799212..c740e64c7ba37 100644 --- a/expression/builtin_time_vec_test.go +++ b/expression/builtin_time_vec_test.go @@ -142,6 +142,9 @@ var vecBuiltinTimeCases = map[string][]vecExprBenchCase{ }, }, }, + ast.Week: { + {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETDatetime, types.ETInt}}, + }, ast.Month: { {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETDatetime}}, }, From 11673d8e71726e72c0a2a33dd0b7010f5a789e8a Mon Sep 17 00:00:00 2001 From: Ping Yu Date: Mon, 18 Nov 2019 12:25:24 +0800 Subject: [PATCH 05/14] expression: implement vectorized evaluation for `builtinDateLiteralSig` (#13502) --- expression/bench_test.go | 14 ++++++++++++-- expression/builtin_time_vec.go | 18 ++++++++++++++++-- expression/builtin_time_vec_test.go | 10 +++++++--- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/expression/bench_test.go b/expression/bench_test.go index eb1a5e5a258df..6df0b4b22de6b 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -693,6 +693,8 @@ type vecExprBenchCase struct { // geners[gen1, gen2] will be regarded as geners[gen1, gen2, nil]. // This field is optional. geners []dataGenerator + // constants are used to generate constant data for children[i]. + constants []*Constant } type vecExprBenchCases map[string][]vecExprBenchCase @@ -788,7 +790,11 @@ func genVecExprBenchCase(ctx sessionctx.Context, funcName string, testCase vecEx input = chunk.New(fts, 1024, 1024) for i, eType := range testCase.childrenTypes { fillColumn(eType, input, i, testCase) - cols[i] = &Column{Index: i, RetType: fts[i]} + if i < len(testCase.constants) && testCase.constants[i] != nil { + cols[i] = testCase.constants[i] + } else { + cols[i] = &Column{Index: i, RetType: fts[i]} + } } expr, err := NewFunction(ctx, funcName, eType2FieldType(testCase.retEvalType), cols...) @@ -920,7 +926,11 @@ func genVecBuiltinFuncBenchCase(ctx sessionctx.Context, funcName string, testCas input = chunk.New(fts, 1024, 1024) for i, eType := range testCase.childrenTypes { fillColumn(eType, input, i, testCase) - cols[i] = &Column{Index: i, RetType: fts[i]} + if i < len(testCase.constants) && testCase.constants[i] != nil { + cols[i] = testCase.constants[i] + } else { + cols[i] = &Column{Index: i, RetType: fts[i]} + } } if len(cols) == 0 { input.SetNumVirtualRows(1024) diff --git a/expression/builtin_time_vec.go b/expression/builtin_time_vec.go index 2a15380cc760c..0ed1bfbb5c260 100644 --- a/expression/builtin_time_vec.go +++ b/expression/builtin_time_vec.go @@ -1986,11 +1986,25 @@ func (b *builtinSubDateStringIntSig) vecEvalTime(input *chunk.Chunk, result *chu } func (b *builtinDateLiteralSig) vectorized() bool { - return false + return true } func (b *builtinDateLiteralSig) vecEvalTime(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + mode := b.ctx.GetSessionVars().SQLMode + if mode.HasNoZeroDateMode() && b.literal.IsZero() { + return types.ErrIncorrectDatetimeValue.GenWithStackByArgs(b.literal.String()) + } + if mode.HasNoZeroInDateMode() && (b.literal.InvalidZero() && !b.literal.IsZero()) { + return types.ErrIncorrectDatetimeValue.GenWithStackByArgs(b.literal.String()) + } + + result.ResizeTime(n, false) + times := result.Times() + for i := range times { + times[i] = b.literal + } + return nil } func (b *builtinTimeLiteralSig) vectorized() bool { diff --git a/expression/builtin_time_vec_test.go b/expression/builtin_time_vec_test.go index c740e64c7ba37..1335288fa64d5 100644 --- a/expression/builtin_time_vec_test.go +++ b/expression/builtin_time_vec_test.go @@ -52,9 +52,13 @@ func (g *unitStrGener) gen() interface{} { } var vecBuiltinTimeCases = map[string][]vecExprBenchCase{ - ast.DateLiteral: {}, - ast.DateDiff: {}, - ast.DateFormat: {}, + ast.DateLiteral: { + {retEvalType: types.ETDatetime, childrenTypes: []types.EvalType{types.ETDatetime}, + constants: []*Constant{{Value: types.NewStringDatum("2019-11-11"), RetType: types.NewFieldType(mysql.TypeString)}}, + }, + }, + ast.DateDiff: {}, + ast.DateFormat: {}, ast.Hour: { {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETDuration}, geners: []dataGenerator{&rangeDurationGener{0.2}}}, }, From f73cd98ccee3346ad820a8ba81ea1e38add677f8 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Sun, 17 Nov 2019 22:36:53 -0600 Subject: [PATCH 06/14] expression: add an argument `EvalType` to method `Column.Reset` to make it safer (#13488) --- expression/bench_test.go | 16 +++++----- expression/builtin_regexp_vec_const_test.go | 2 +- expression/builtin_time_vec_generated_test.go | 8 ++--- expression/builtin_vectorized_test.go | 16 +++++----- expression/generator/time_vec.go | 14 +++++++-- expression/vectorized.go | 2 +- util/chunk/chunk.go | 2 +- util/chunk/column.go | 30 +++++++++++++++++-- util/chunk/column_test.go | 17 +++++++++++ 9 files changed, 79 insertions(+), 28 deletions(-) diff --git a/expression/bench_test.go b/expression/bench_test.go index 6df0b4b22de6b..55181308ffd38 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -714,7 +714,7 @@ func fillColumnWithGener(eType types.EvalType, chk *chunk.Chunk, colIdx int, gen } col := chk.Column(colIdx) - col.Reset() + col.Reset(eType) for i := 0; i < batchSize; i++ { v := gen.gen() if v == nil { @@ -1284,7 +1284,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases switch testCase.retEvalType { case types.ETInt: for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(testCase.retEvalType) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := baseFunc.evalInt(row) if err != nil { @@ -1299,7 +1299,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases } case types.ETReal: for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(testCase.retEvalType) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := baseFunc.evalReal(row) if err != nil { @@ -1314,7 +1314,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases } case types.ETDecimal: for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(testCase.retEvalType) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := baseFunc.evalDecimal(row) if err != nil { @@ -1329,7 +1329,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases } case types.ETDatetime, types.ETTimestamp: for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(testCase.retEvalType) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := baseFunc.evalTime(row) if err != nil { @@ -1344,7 +1344,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases } case types.ETDuration: for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(testCase.retEvalType) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := baseFunc.evalDuration(row) if err != nil { @@ -1359,7 +1359,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases } case types.ETJson: for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(testCase.retEvalType) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := baseFunc.evalJSON(row) if err != nil { @@ -1374,7 +1374,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases } case types.ETString: for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(testCase.retEvalType) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := baseFunc.evalString(row) if err != nil { diff --git a/expression/builtin_regexp_vec_const_test.go b/expression/builtin_regexp_vec_const_test.go index 28f21e6a64991..b7ec27a2248f0 100644 --- a/expression/builtin_regexp_vec_const_test.go +++ b/expression/builtin_regexp_vec_const_test.go @@ -94,7 +94,7 @@ func BenchmarkVectorizedBuiltinRegexpForConstants(b *testing.B) { b.ResetTimer() it := chunk.NewIterator4Chunk(input) for i := 0; i < b.N; i++ { - output.Reset() + output.Reset(types.ETInt) for row := it.Begin(); row != it.End(); row = it.Next() { v, isNull, err := bf.evalInt(row) if err != nil { diff --git a/expression/builtin_time_vec_generated_test.go b/expression/builtin_time_vec_generated_test.go index b024cc2a531df..60dff916aca61 100644 --- a/expression/builtin_time_vec_generated_test.go +++ b/expression/builtin_time_vec_generated_test.go @@ -102,20 +102,20 @@ var vecBuiltinTimeGeneratedCases = map[string][]vecExprBenchCase{ // builtinAddDateAndDurationSig { retEvalType: types.ETString, - childrenTypes: []types.EvalType{types.ETDuration, types.ETDuration}, + childrenTypes: []types.EvalType{types.ETDatetime, types.ETDuration}, childrenFieldTypes: []*types.FieldType{types.NewFieldType(mysql.TypeDate), types.NewFieldType(mysql.TypeDuration)}, geners: []dataGenerator{ - gener{defaultGener{eType: types.ETDuration, nullRation: 0.2}}, + gener{defaultGener{eType: types.ETDatetime, nullRation: 0.2}}, gener{defaultGener{eType: types.ETDuration, nullRation: 0.2}}, }, }, // builtinAddDateAndStringSig { retEvalType: types.ETString, - childrenTypes: []types.EvalType{types.ETDuration, types.ETString}, + childrenTypes: []types.EvalType{types.ETDatetime, types.ETString}, childrenFieldTypes: []*types.FieldType{types.NewFieldType(mysql.TypeDate), types.NewFieldType(mysql.TypeString)}, geners: []dataGenerator{ - gener{defaultGener{eType: types.ETDuration, nullRation: 0.2}}, + gener{defaultGener{eType: types.ETDatetime, nullRation: 0.2}}, gener{defaultGener{eType: types.ETString, nullRation: 0.2}}, }, }, diff --git a/expression/builtin_vectorized_test.go b/expression/builtin_vectorized_test.go index 2e9f7de868cad..552fbce052a5d 100644 --- a/expression/builtin_vectorized_test.go +++ b/expression/builtin_vectorized_test.go @@ -543,7 +543,7 @@ func (s *testEvaluatorSuite) TestDoubleVec2Row(c *C) { eTypes := []types.EvalType{types.ETInt, types.ETReal, types.ETDecimal, types.ETDuration, types.ETString, types.ETDatetime, types.ETJson} for _, eType := range eTypes { rowDouble, input, result, err := genMockRowDouble(eType, true) - result.Reset() + result.Reset(eType) c.Assert(err, IsNil) it := chunk.NewIterator4Chunk(input) for row := it.Begin(); row != it.End(); row = it.Next() { @@ -586,7 +586,7 @@ func evalRows(b *testing.B, it *chunk.Iterator4Chunk, eType types.EvalType, resu switch eType { case types.ETInt: for i := 0; i < b.N; i++ { - result.Reset() + result.Reset(eType) for r := it.Begin(); r != it.End(); r = it.Next() { v, isNull, err := rowDouble.evalInt(r) if err != nil { @@ -601,7 +601,7 @@ func evalRows(b *testing.B, it *chunk.Iterator4Chunk, eType types.EvalType, resu } case types.ETReal: for i := 0; i < b.N; i++ { - result.Reset() + result.Reset(eType) for r := it.Begin(); r != it.End(); r = it.Next() { v, isNull, err := rowDouble.evalReal(r) if err != nil { @@ -616,7 +616,7 @@ func evalRows(b *testing.B, it *chunk.Iterator4Chunk, eType types.EvalType, resu } case types.ETDecimal: for i := 0; i < b.N; i++ { - result.Reset() + result.Reset(eType) for r := it.Begin(); r != it.End(); r = it.Next() { v, isNull, err := rowDouble.evalDecimal(r) if err != nil { @@ -631,7 +631,7 @@ func evalRows(b *testing.B, it *chunk.Iterator4Chunk, eType types.EvalType, resu } case types.ETDuration: for i := 0; i < b.N; i++ { - result.Reset() + result.Reset(eType) for r := it.Begin(); r != it.End(); r = it.Next() { v, isNull, err := rowDouble.evalDuration(r) if err != nil { @@ -646,7 +646,7 @@ func evalRows(b *testing.B, it *chunk.Iterator4Chunk, eType types.EvalType, resu } case types.ETString: for i := 0; i < b.N; i++ { - result.Reset() + result.Reset(eType) for r := it.Begin(); r != it.End(); r = it.Next() { v, isNull, err := rowDouble.evalString(r) if err != nil { @@ -661,7 +661,7 @@ func evalRows(b *testing.B, it *chunk.Iterator4Chunk, eType types.EvalType, resu } case types.ETDatetime: for i := 0; i < b.N; i++ { - result.Reset() + result.Reset(eType) for r := it.Begin(); r != it.End(); r = it.Next() { v, isNull, err := rowDouble.evalTime(r) if err != nil { @@ -676,7 +676,7 @@ func evalRows(b *testing.B, it *chunk.Iterator4Chunk, eType types.EvalType, resu } case types.ETJson: for i := 0; i < b.N; i++ { - result.Reset() + result.Reset(eType) for r := it.Begin(); r != it.End(); r = it.Next() { v, isNull, err := rowDouble.evalJSON(r) if err != nil { diff --git a/expression/generator/time_vec.go b/expression/generator/time_vec.go index 9b39602e31403..cd03ec9dbd8b7 100644 --- a/expression/generator/time_vec.go +++ b/expression/generator/time_vec.go @@ -472,13 +472,22 @@ var vecBuiltin{{.Category}}GeneratedCases = map[string][]vecExprBenchCase{ {{ range .Sigs }} // {{ .SigName }} { retEvalType: types.ET{{ .Output.ETName }}, + {{- if eq .TestTypeA "" }} childrenTypes: []types.EvalType{types.ET{{ .TypeA.ETName }}, types.ET{{ .TypeB.ETName }}}, + {{- else }} + childrenTypes: []types.EvalType{types.ET{{ .TestTypeA }}, types.ET{{ .TestTypeB }}}, + {{- end }} {{- if ne .FieldTypeA "" }} childrenFieldTypes: []*types.FieldType{types.NewFieldType(mysql.Type{{.FieldTypeA}}), types.NewFieldType(mysql.Type{{.FieldTypeB}})}, {{- end }} geners: []dataGenerator{ + {{- if eq .TestTypeA "" }} gener{defaultGener{eType: types.ET{{.TypeA.ETName}}, nullRation: 0.2}}, gener{defaultGener{eType: types.ET{{.TypeB.ETName}}, nullRation: 0.2}}, + {{- else }} + gener{defaultGener{eType: types.ET{{ .TestTypeA }}, nullRation: 0.2}}, + gener{defaultGener{eType: types.ET{{ .TestTypeB }}, nullRation: 0.2}}, + {{- end }} }, }, {{ end }} @@ -547,8 +556,8 @@ var addTimeSigsTmpl = []sig{ {SigName: "builtinAddDurationAndStringSig", TypeA: TypeDuration, TypeB: TypeString, Output: TypeDuration}, {SigName: "builtinAddStringAndDurationSig", TypeA: TypeString, TypeB: TypeDuration, Output: TypeString}, {SigName: "builtinAddStringAndStringSig", TypeA: TypeString, TypeB: TypeString, Output: TypeString}, - {SigName: "builtinAddDateAndDurationSig", TypeA: TypeDuration, TypeB: TypeDuration, Output: TypeString, FieldTypeA: "Date", FieldTypeB: "Duration"}, - {SigName: "builtinAddDateAndStringSig", TypeA: TypeDuration, TypeB: TypeString, Output: TypeString, FieldTypeA: "Date", FieldTypeB: "String"}, + {SigName: "builtinAddDateAndDurationSig", TypeA: TypeDuration, TypeB: TypeDuration, Output: TypeString, FieldTypeA: "Date", FieldTypeB: "Duration", TestTypeA: "Datetime", TestTypeB: "Duration"}, + {SigName: "builtinAddDateAndStringSig", TypeA: TypeDuration, TypeB: TypeString, Output: TypeString, FieldTypeA: "Date", FieldTypeB: "String", TestTypeA: "Datetime", TestTypeB: "String"}, {SigName: "builtinAddTimeDateTimeNullSig", TypeA: TypeDatetime, TypeB: TypeDatetime, Output: TypeDatetime, AllNull: true}, {SigName: "builtinAddTimeStringNullSig", TypeA: TypeDatetime, TypeB: TypeDatetime, Output: TypeString, AllNull: true, FieldTypeA: "Date", FieldTypeB: "Datetime"}, @@ -570,6 +579,7 @@ type sig struct { SigName string TypeA, TypeB, Output TypeContext FieldTypeA, FieldTypeB string // Optional + TestTypeA, TestTypeB string // Optional, specific Type for test in builtinAddDateAndDurationSig & builtinAddDateAndStringSig AllNull bool } diff --git a/expression/vectorized.go b/expression/vectorized.go index 19c1fba59605c..6abf7773398b2 100644 --- a/expression/vectorized.go +++ b/expression/vectorized.go @@ -25,7 +25,7 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, targetType typ if input != nil { n = input.NumRows() if n == 0 { - result.Reset() + result.Reset(targetType) return nil } } diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 83118932f8061..fbcb0ade9f5ae 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -249,7 +249,7 @@ func (c *Chunk) Reset() { return } for _, col := range c.columns { - col.Reset() + col.reset() } c.numVirtualRows = 0 } diff --git a/util/chunk/column.go b/util/chunk/column.go index 36b2a9cffc2ef..cc4fa82cddd28 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -14,6 +14,7 @@ package chunk import ( + "fmt" "math/bits" "reflect" "time" @@ -91,8 +92,31 @@ func (c *Column) isFixed() bool { return c.elemBuf != nil } -// Reset resets this Column. -func (c *Column) Reset() { +// Reset resets this Column according to the EvalType. +// Different from reset, Reset will reset the elemBuf. +func (c *Column) Reset(eType types.EvalType) { + switch eType { + case types.ETInt: + c.ResizeInt64(0, false) + case types.ETReal: + c.ResizeFloat64(0, false) + case types.ETDecimal: + c.ResizeDecimal(0, false) + case types.ETString: + c.ReserveString(0) + case types.ETDatetime, types.ETTimestamp: + c.ResizeTime(0, false) + case types.ETDuration: + c.ResizeGoDuration(0, false) + case types.ETJson: + c.ReserveJSON(0) + default: + panic(fmt.Sprintf("invalid EvalType %v", eType)) + } +} + +// reset resets the underlying data of this Column but doesn't modify its data type. +func (c *Column) reset() { c.length = 0 c.nullBitmap = c.nullBitmap[:0] if len(c.offsets) > 0 { @@ -639,7 +663,7 @@ func (c *Column) CopyReconstruct(sel []int, dst *Column) *Column { if dst == nil { dst = newColumn(c.typeSize(), len(sel)) } else { - dst.Reset() + dst.reset() } if c.isFixed() { diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index 2d5178c786475..f2bb8ddc09a3c 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -978,6 +978,23 @@ func (s *testChunkSuite) TestVectorizedNulls(c *check.C) { } } +func (s *testChunkSuite) TestResetColumn(c *check.C) { + col0 := NewColumn(types.NewFieldType(mysql.TypeVarString), 0) + col1 := NewColumn(types.NewFieldType(mysql.TypeLonglong), 0) + + // using col0.reset() here will cause panic since it doesn't reset the elemBuf field which + // is used by MergeNulls. + col0.Reset(types.ETInt) + col0.MergeNulls(col1) + + col := NewColumn(types.NewFieldType(mysql.TypeDatetime), 0) + col.Reset(types.ETDuration) + col.AppendDuration(types.Duration{}) + // using col.reset() above will let this assertion fail since the length of initialized elemBuf + // is sizeTime. + c.Assert(len(col.data), check.Equals, sizeGoDuration) +} + func BenchmarkMergeNullsVectorized(b *testing.B) { cols := genNullCols(3) b.ResetTimer() From ee1ac01282f2f2ec5f5c9561fb4eff15dedf4698 Mon Sep 17 00:00:00 2001 From: "Zhuomin(Charming) Liu" Date: Mon, 18 Nov 2019 13:34:24 +0800 Subject: [PATCH 07/14] planner, executor: fix wrong result in index merge join. (#13192) --- executor/builder.go | 15 ++++++----- executor/index_lookup_merge_join.go | 36 +++++++++++++++----------- executor/join_test.go | 20 ++++++++++++++ planner/core/exhaust_physical_plans.go | 33 +++++++++++++++++------ planner/core/physical_plans.go | 2 ++ 5 files changed, 76 insertions(+), 30 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index b16d76a9abb2c..bb9dc9959ce34 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1847,13 +1847,14 @@ func (b *executorBuilder) buildIndexLookUpMergeJoin(v *plannercore.PhysicalIndex compareFuncs: v.OuterCompareFuncs, }, innerMergeCtx: innerMergeCtx{ - readerBuilder: &dataReaderBuilder{Plan: innerPlan, executorBuilder: b}, - rowTypes: innerTypes, - joinKeys: v.InnerJoinKeys, - keyCols: innerKeyCols, - compareFuncs: v.CompareFuncs, - colLens: v.IdxColLens, - desc: v.Desc, + readerBuilder: &dataReaderBuilder{Plan: innerPlan, executorBuilder: b}, + rowTypes: innerTypes, + joinKeys: v.InnerJoinKeys, + keyCols: innerKeyCols, + compareFuncs: v.CompareFuncs, + colLens: v.IdxColLens, + desc: v.Desc, + keyOff2KeyOffOrderByIdx: v.KeyOff2KeyOffOrderByIdx, }, workerWg: new(sync.WaitGroup), isOuterJoin: v.JoinType.IsOuterJoin(), diff --git a/executor/index_lookup_merge_join.go b/executor/index_lookup_merge_join.go index e4dbe7369d3b2..f6212e83489d9 100644 --- a/executor/index_lookup_merge_join.go +++ b/executor/index_lookup_merge_join.go @@ -76,13 +76,14 @@ type outerMergeCtx struct { } type innerMergeCtx struct { - readerBuilder *dataReaderBuilder - rowTypes []*types.FieldType - joinKeys []*expression.Column - keyCols []int - compareFuncs []expression.CompareFunc - colLens []int - desc bool + readerBuilder *dataReaderBuilder + rowTypes []*types.FieldType + joinKeys []*expression.Column + keyCols []int + compareFuncs []expression.CompareFunc + colLens []int + desc bool + keyOff2KeyOffOrderByIdx []int } type lookUpMergeJoinTask struct { @@ -423,15 +424,20 @@ func (imw *innerMergeWorker) handleTask(ctx context.Context, task *lookUpMergeJo sort.Slice(task.outerOrderIdx, func(i, j int) bool { idxI, idxJ := task.outerOrderIdx[i], task.outerOrderIdx[j] rowI, rowJ := task.outerResult.GetRow(idxI), task.outerResult.GetRow(idxJ) - for id, joinKey := range imw.outerMergeCtx.joinKeys { - cmp, _, err := imw.outerMergeCtx.compareFuncs[id](imw.ctx, joinKey, joinKey, rowI, rowJ) + var cmp int64 + var err error + for _, keyOff := range imw.keyOff2KeyOffOrderByIdx { + joinKey := imw.outerMergeCtx.joinKeys[keyOff] + cmp, _, err = imw.outerMergeCtx.compareFuncs[keyOff](imw.ctx, joinKey, joinKey, rowI, rowJ) terror.Log(err) - if cmp != 0 || imw.nextColCompareFilters == nil { - return cmp < 0 + if cmp != 0 { + break } - return imw.nextColCompareFilters.CompareRow(rowI, rowJ) < 0 } - return false + if cmp != 0 || imw.nextColCompareFilters == nil { + return cmp < 0 + } + return imw.nextColCompareFilters.CompareRow(rowI, rowJ) < 0 }) } dLookUpKeys, err := imw.constructDatumLookupKeys(task) @@ -571,8 +577,8 @@ func (imw *innerMergeWorker) fetchInnerRowsWithSameKey(ctx context.Context, task } func (imw *innerMergeWorker) compare(outerRow, innerRow chunk.Row) (int, error) { - for i := 0; i < len(imw.outerMergeCtx.joinKeys); i++ { - cmp, _, err := imw.innerMergeCtx.compareFuncs[i](imw.ctx, imw.outerMergeCtx.joinKeys[i], imw.innerMergeCtx.joinKeys[i], outerRow, innerRow) + for _, keyOff := range imw.innerMergeCtx.keyOff2KeyOffOrderByIdx { + cmp, _, err := imw.innerMergeCtx.compareFuncs[keyOff](imw.ctx, imw.outerMergeCtx.joinKeys[keyOff], imw.innerMergeCtx.joinKeys[keyOff], outerRow, innerRow) if err != nil || cmp != 0 { return int(cmp), err } diff --git a/executor/join_test.go b/executor/join_test.go index 426d0744abd3c..b5130da2d461d 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -1589,3 +1589,23 @@ func (s *testSuiteJoin1) TestIssue11390(c *C) { tk.MustExec("insert into 11390t values(1, 1)") tk.MustQuery("select /*+ TIDB_INLJ(t1, t2) */ * from 11390t t1, 11390t t2 where t1.k2 > 0 and t1.k2 = t2.k2 and t2.k1=1;").Check(testkit.Rows("1 1 1 1")) } + +func (s *testSuiteJoin1) TestIssue13177(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(a varchar(20), b int, c int)") + tk.MustExec("create table t2(a varchar(20), b int, c int, primary key(a, b))") + tk.MustExec("insert into t1 values(\"abcd\", 1, 1), (\"bacd\", 2, 2), (\"cbad\", 3, 3)") + tk.MustExec("insert into t2 values(\"bcd\", 1, 1), (\"acd\", 2, 2), (\"bad\", 3, 3)") + tk.MustQuery("select /*+ inl_merge_join(t1, t2) */ * from t1 join t2 on substr(t1.a, 2, 4) = t2.a and t1.b = t2.b where t1.c between 1 and 5").Check(testkit.Rows( + "bacd 2 2 acd 2 2", + "cbad 3 3 bad 3 3", + "abcd 1 1 bcd 1 1", + )) + tk.MustQuery("select /*+ inl_merge_join(t1, t2) */ t1.* from t1 join t2 on substr(t1.a, 2, 4) = t2.a and t1.b = t2.b where t1.c between 1 and 5").Check(testkit.Rows( + "bacd 2 2", + "cbad 3 3", + "abcd 1 1", + )) +} diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index edc1336ee3a4e..1c1abb51b0a49 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -17,6 +17,7 @@ import ( "bytes" "fmt" "math" + "sort" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" @@ -399,12 +400,27 @@ func (p *LogicalJoin) constructIndexMergeJoin( if hasPrefixCol { continue } + + // keyOff2KeyOffOrderByIdx is map the join keys offsets to [0, len(joinKeys)) ordered by the + // join key position in inner index. + keyOff2KeyOffOrderByIdx := make([]int, len(join.OuterJoinKeys)) + keyOffMapList := make([]int, len(join.KeyOff2IdxOff)) + copy(keyOffMapList, join.KeyOff2IdxOff) + keyOffMap := make(map[int]int) + for i, idxOff := range keyOffMapList { + keyOffMap[idxOff] = i + } + sort.Slice(keyOffMapList, func(i, j int) bool { return keyOffMapList[i] < keyOffMapList[j] }) + for keyOff, idxOff := range keyOffMapList { + keyOff2KeyOffOrderByIdx[keyOffMap[idxOff]] = keyOff + } // isOuterKeysPrefix means whether the outer join keys are the prefix of the prop items. isOuterKeysPrefix := len(join.OuterJoinKeys) <= len(prop.Items) compareFuncs := make([]expression.CompareFunc, 0, len(join.OuterJoinKeys)) outerCompareFuncs := make([]expression.CompareFunc, 0, len(join.OuterJoinKeys)) - for i := range join.OuterJoinKeys { - if isOuterKeysPrefix && !prop.Items[i].Col.Equal(nil, join.OuterJoinKeys[i]) { + + for i := range join.KeyOff2IdxOff { + if isOuterKeysPrefix && !prop.Items[i].Col.Equal(nil, join.OuterJoinKeys[keyOff2KeyOffOrderByIdx[i]]) { isOuterKeysPrefix = false } compareFuncs = append(compareFuncs, expression.GetCmpFunction(join.OuterJoinKeys[i], join.InnerJoinKeys[i])) @@ -413,7 +429,7 @@ func (p *LogicalJoin) constructIndexMergeJoin( // canKeepOuterOrder means whether the prop items are the prefix of the outer join keys. canKeepOuterOrder := len(prop.Items) <= len(join.OuterJoinKeys) for i := 0; canKeepOuterOrder && i < len(prop.Items); i++ { - if !prop.Items[i].Col.Equal(nil, join.OuterJoinKeys[i]) { + if !prop.Items[i].Col.Equal(nil, join.OuterJoinKeys[keyOff2KeyOffOrderByIdx[i]]) { canKeepOuterOrder = false } } @@ -422,11 +438,12 @@ func (p *LogicalJoin) constructIndexMergeJoin( // `isOuterKeysPrefix` to be true. if canKeepOuterOrder || isOuterKeysPrefix { indexMergeJoin := PhysicalIndexMergeJoin{ - PhysicalIndexJoin: *join, - NeedOuterSort: !isOuterKeysPrefix, - CompareFuncs: compareFuncs, - OuterCompareFuncs: outerCompareFuncs, - Desc: !prop.IsEmpty() && prop.Items[0].Desc, + PhysicalIndexJoin: *join, + KeyOff2KeyOffOrderByIdx: keyOff2KeyOffOrderByIdx, + NeedOuterSort: !isOuterKeysPrefix, + CompareFuncs: compareFuncs, + OuterCompareFuncs: outerCompareFuncs, + Desc: !prop.IsEmpty() && prop.Items[0].Desc, }.Init(p.ctx) indexMergeJoins = append(indexMergeJoins, indexMergeJoin) } diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 47ac67b9da723..0028f97faad8a 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -315,6 +315,8 @@ type PhysicalIndexJoin struct { type PhysicalIndexMergeJoin struct { PhysicalIndexJoin + // KeyOff2KeyOffOrderByIdx maps the offsets in join keys to the offsets in join keys order by index. + KeyOff2KeyOffOrderByIdx []int // NeedOuterSort means whether outer rows should be sorted to build range. NeedOuterSort bool // CompareFuncs store the compare functions for outer join keys and inner join key. From 66d99a1eab54f053da0825ba3ee57f4b283370e5 Mon Sep 17 00:00:00 2001 From: shihongzhi Date: Mon, 18 Nov 2019 14:12:37 +0800 Subject: [PATCH 08/14] expression: implement vectorized evaluation for `builtinCurrentUserSig` (#13356) --- expression/bench_test.go | 4 ++-- expression/builtin_info_vec.go | 16 ++++++++++++++-- expression/builtin_info_vec_test.go | 6 ++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/expression/bench_test.go b/expression/bench_test.go index 55181308ffd38..b8e60d2fa9f95 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -1006,7 +1006,7 @@ func testVectorizedBuiltinFunc(c *C, vecExprCases vecExprBenchCases) { err := ctx.GetSessionVars().SetSystemVar(variable.BlockEncryptionMode, "aes-128-ecb") c.Assert(err, IsNil) } - if funcName == ast.User { + if funcName == ast.CurrentUser || funcName == ast.User { ctx.GetSessionVars().User = &auth.UserIdentity{ Username: "tidb", Hostname: "localhost", @@ -1211,7 +1211,7 @@ func benchmarkVectorizedBuiltinFunc(b *testing.B, vecExprCases vecExprBenchCases panic(err) } } - if funcName == ast.User { + if funcName == ast.CurrentUser || funcName == ast.User { ctx.GetSessionVars().User = &auth.UserIdentity{ Username: "tidb", Hostname: "localhost", diff --git a/expression/builtin_info_vec.go b/expression/builtin_info_vec.go index e72f6278399d8..42638fcf76c9f 100644 --- a/expression/builtin_info_vec.go +++ b/expression/builtin_info_vec.go @@ -98,11 +98,23 @@ func (b *builtinRowCountSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column } func (b *builtinCurrentUserSig) vectorized() bool { - return false + return true } +// evalString evals a builtinCurrentUserSig. +// See https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_current-user func (b *builtinCurrentUserSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + + data := b.ctx.GetSessionVars() + result.ReserveString(n) + if data == nil || data.User == nil { + return errors.Errorf("Missing session variable when eval builtin") + } + for i := 0; i < n; i++ { + result.AppendString(data.User.AuthIdentityString()) + } + return nil } func (b *builtinCurrentRoleSig) vectorized() bool { diff --git a/expression/builtin_info_vec_test.go b/expression/builtin_info_vec_test.go index 3d89c4f0e033c..017dc75fac36b 100644 --- a/expression/builtin_info_vec_test.go +++ b/expression/builtin_info_vec_test.go @@ -50,8 +50,10 @@ var vecBuiltinInfoCases = map[string][]vecExprBenchCase{ ast.TiDBVersion: { {retEvalType: types.ETString, childrenTypes: []types.EvalType{}}, }, - ast.CurrentUser: {}, - ast.FoundRows: {}, + ast.CurrentUser: { + {retEvalType: types.ETString, childrenTypes: []types.EvalType{}}, + }, + ast.FoundRows: {}, ast.Database: { {retEvalType: types.ETString, childrenTypes: []types.EvalType{}}, }, From 0b41fe9d7110a1ef41cfc0e2e3aa67bce3c78969 Mon Sep 17 00:00:00 2001 From: Ping Yu Date: Mon, 18 Nov 2019 14:16:34 +0800 Subject: [PATCH 09/14] expression: implement vectorized evaluation for `builtinCastStringAsRealSig` (#13445) --- expression/bench_test.go | 13 ++++++++++ expression/builtin_cast_vec.go | 40 +++++++++++++++++++++++++++-- expression/builtin_cast_vec_test.go | 7 +++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/expression/bench_test.go b/expression/bench_test.go index b8e60d2fa9f95..a983305ce8a3d 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -406,6 +406,19 @@ func (g *numStrGener) gen() interface{} { return fmt.Sprintf("%v", g.rangeInt64Gener.gen()) } +// realStrGener is used to generate real number strings. +type realStrGener struct { + rangeRealGener +} + +func (g *realStrGener) gen() interface{} { + val := g.rangeRealGener.gen() + if val == nil { + return nil + } + return fmt.Sprintf("%v", val) +} + // ipv6StrGener is used to generate ipv6 strings. type ipv6StrGener struct { } diff --git a/expression/builtin_cast_vec.go b/expression/builtin_cast_vec.go index 0358bfe094cba..897af3e904c9e 100644 --- a/expression/builtin_cast_vec.go +++ b/expression/builtin_cast_vec.go @@ -1240,11 +1240,47 @@ func (b *builtinCastJSONAsDecimalSig) vecEvalDecimal(input *chunk.Chunk, result } func (b *builtinCastStringAsRealSig) vectorized() bool { - return false + return true } func (b *builtinCastStringAsRealSig) vecEvalReal(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + if IsBinaryLiteral(b.args[0]) { + // This block is skipped by `castAsRealFunctionClass.getFunction()` + return b.args[0].VecEvalReal(b.ctx, input, result) + } + + n := input.NumRows() + bufStrings, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(bufStrings) + if err := b.args[0].VecEvalString(b.ctx, input, bufStrings); err != nil { + return err + } + + result.ResizeFloat64(n, false) + result.MergeNulls(bufStrings) + f64s := result.Float64s() + sc := b.ctx.GetSessionVars().StmtCtx + for i := 0; i < n; i++ { + if result.IsNull(i) { + continue + } + val := bufStrings.GetString(i) + res, err := types.StrToFloat(sc, val) + if err != nil { + return err + } + if b.inUnion && mysql.HasUnsignedFlag(b.tp.Flag) && res < 0 { + res = 0 + } + if res, err = types.ProduceFloatWithSpecifiedTp(res, b.tp, sc); err != nil { + return err + } + f64s[i] = res + } + return nil } func (b *builtinCastStringAsDecimalSig) vectorized() bool { diff --git a/expression/builtin_cast_vec_test.go b/expression/builtin_cast_vec_test.go index 5033506fd4a96..b96a7979b72ef 100644 --- a/expression/builtin_cast_vec_test.go +++ b/expression/builtin_cast_vec_test.go @@ -20,6 +20,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" ) @@ -38,6 +39,12 @@ var vecBuiltinCastCases = map[string][]vecExprBenchCase{ {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETJson}}, {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETDecimal}}, {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETDatetime}}, + {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETString}, + geners: []dataGenerator{&realStrGener{rangeRealGener{begin: -100000.0, end: 100000.0, nullRation: 0.5}}}, + }, + {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETString}, + constants: []*Constant{{Value: types.NewBinaryLiteralDatum([]byte("TiDB")), RetType: types.NewFieldType(mysql.TypeVarString)}}, + }, {retEvalType: types.ETDuration, childrenTypes: []types.EvalType{types.ETDatetime}, geners: []dataGenerator{&dateTimeGenerWithFsp{ defaultGener: defaultGener{nullRation: 0.2, eType: types.ETDatetime}, From 381e745b1d8dcc6b6723866514b8758d76b4f734 Mon Sep 17 00:00:00 2001 From: LinJs9 <30310519+Baytwt@users.noreply.github.com> Date: Mon, 18 Nov 2019 14:20:47 +0800 Subject: [PATCH 10/14] expression: implement vectorized evaluation for `builtinNameConstJSONSig` (#13384) --- expression/builtin_miscellaneous_vec.go | 4 ++-- expression/builtin_miscellaneous_vec_test.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/expression/builtin_miscellaneous_vec.go b/expression/builtin_miscellaneous_vec.go index 22892f7a7a176..d60aa1ce4e279 100644 --- a/expression/builtin_miscellaneous_vec.go +++ b/expression/builtin_miscellaneous_vec.go @@ -324,11 +324,11 @@ func (b *builtinNameConstDecimalSig) vecEvalDecimal(input *chunk.Chunk, result * } func (b *builtinNameConstJSONSig) vectorized() bool { - return false + return true } func (b *builtinNameConstJSONSig) vecEvalJSON(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + return b.args[1].VecEvalJSON(b.ctx, input, result) } func (b *builtinInet6AtonSig) vectorized() bool { diff --git a/expression/builtin_miscellaneous_vec_test.go b/expression/builtin_miscellaneous_vec_test.go index 3420e27285d49..841bacc3e63a4 100644 --- a/expression/builtin_miscellaneous_vec_test.go +++ b/expression/builtin_miscellaneous_vec_test.go @@ -80,6 +80,7 @@ var vecBuiltinMiscellaneousCases = map[string][]vecExprBenchCase{ {retEvalType: types.ETDecimal, childrenTypes: []types.EvalType{types.ETString, types.ETDecimal}}, {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETString, types.ETInt}}, {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETString, types.ETReal}}, + {retEvalType: types.ETJson, childrenTypes: []types.EvalType{types.ETString, types.ETJson}}, {retEvalType: types.ETTimestamp, childrenTypes: []types.EvalType{types.ETString, types.ETTimestamp}}, }, } From 839ea6f06b1d6513cc52cab20a4fdf9ae15bf209 Mon Sep 17 00:00:00 2001 From: Wenxuan Date: Mon, 18 Nov 2019 16:50:41 +0800 Subject: [PATCH 11/14] Revert "expression: implement vectorized evaluation for `builtinCastStringAsRealSig` (#13445)" (#13543) --- expression/bench_test.go | 13 ---------- expression/builtin_cast_vec.go | 40 ++--------------------------- expression/builtin_cast_vec_test.go | 7 ----- 3 files changed, 2 insertions(+), 58 deletions(-) diff --git a/expression/bench_test.go b/expression/bench_test.go index a983305ce8a3d..b8e60d2fa9f95 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -406,19 +406,6 @@ func (g *numStrGener) gen() interface{} { return fmt.Sprintf("%v", g.rangeInt64Gener.gen()) } -// realStrGener is used to generate real number strings. -type realStrGener struct { - rangeRealGener -} - -func (g *realStrGener) gen() interface{} { - val := g.rangeRealGener.gen() - if val == nil { - return nil - } - return fmt.Sprintf("%v", val) -} - // ipv6StrGener is used to generate ipv6 strings. type ipv6StrGener struct { } diff --git a/expression/builtin_cast_vec.go b/expression/builtin_cast_vec.go index 897af3e904c9e..0358bfe094cba 100644 --- a/expression/builtin_cast_vec.go +++ b/expression/builtin_cast_vec.go @@ -1240,47 +1240,11 @@ func (b *builtinCastJSONAsDecimalSig) vecEvalDecimal(input *chunk.Chunk, result } func (b *builtinCastStringAsRealSig) vectorized() bool { - return true + return false } func (b *builtinCastStringAsRealSig) vecEvalReal(input *chunk.Chunk, result *chunk.Column) error { - if IsBinaryLiteral(b.args[0]) { - // This block is skipped by `castAsRealFunctionClass.getFunction()` - return b.args[0].VecEvalReal(b.ctx, input, result) - } - - n := input.NumRows() - bufStrings, err := b.bufAllocator.get(types.ETString, n) - if err != nil { - return err - } - defer b.bufAllocator.put(bufStrings) - if err := b.args[0].VecEvalString(b.ctx, input, bufStrings); err != nil { - return err - } - - result.ResizeFloat64(n, false) - result.MergeNulls(bufStrings) - f64s := result.Float64s() - sc := b.ctx.GetSessionVars().StmtCtx - for i := 0; i < n; i++ { - if result.IsNull(i) { - continue - } - val := bufStrings.GetString(i) - res, err := types.StrToFloat(sc, val) - if err != nil { - return err - } - if b.inUnion && mysql.HasUnsignedFlag(b.tp.Flag) && res < 0 { - res = 0 - } - if res, err = types.ProduceFloatWithSpecifiedTp(res, b.tp, sc); err != nil { - return err - } - f64s[i] = res - } - return nil + return errors.Errorf("not implemented") } func (b *builtinCastStringAsDecimalSig) vectorized() bool { diff --git a/expression/builtin_cast_vec_test.go b/expression/builtin_cast_vec_test.go index b96a7979b72ef..5033506fd4a96 100644 --- a/expression/builtin_cast_vec_test.go +++ b/expression/builtin_cast_vec_test.go @@ -20,7 +20,6 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" ) @@ -39,12 +38,6 @@ var vecBuiltinCastCases = map[string][]vecExprBenchCase{ {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETJson}}, {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETDecimal}}, {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETDatetime}}, - {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETString}, - geners: []dataGenerator{&realStrGener{rangeRealGener{begin: -100000.0, end: 100000.0, nullRation: 0.5}}}, - }, - {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETString}, - constants: []*Constant{{Value: types.NewBinaryLiteralDatum([]byte("TiDB")), RetType: types.NewFieldType(mysql.TypeVarString)}}, - }, {retEvalType: types.ETDuration, childrenTypes: []types.EvalType{types.ETDatetime}, geners: []dataGenerator{&dateTimeGenerWithFsp{ defaultGener: defaultGener{nullRation: 0.2, eType: types.ETDatetime}, From 8401cbdeba76e1dc7ec2b825d60fae8fad7ecbf9 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Mon, 18 Nov 2019 18:13:50 +0800 Subject: [PATCH 12/14] store: update kvproto.CheckTxnStatus response (#13432) --- go.mod | 2 +- go.sum | 9 +++- store/mockstore/mocktikv/mock_tikv_test.go | 34 ++++++++---- store/mockstore/mocktikv/mvcc.go | 2 +- store/mockstore/mocktikv/mvcc_leveldb.go | 61 +++++++++++++--------- store/mockstore/mocktikv/rpc.go | 4 +- store/tikv/lock_resolver.go | 4 +- store/tikv/lock_test.go | 14 +++-- 8 files changed, 85 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index d855de8222b7a..ac464b952e60c 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e - github.com/pingcap/kvproto v0.0.0-20191104103048-40f562012fb1 + github.com/pingcap/kvproto v0.0.0-20191118050206-47672e7eabc0 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 github.com/pingcap/parser v0.0.0-20191112053614-3b43b46331d5 github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 diff --git a/go.sum b/go.sum index 51470197e9643..6e55be6ac3560 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,13 @@ github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= -github.com/pingcap/kvproto v0.0.0-20191104103048-40f562012fb1 h1:J5oimSv+0emw5e/D1ZX/zh2WcMv0pOVT9QKruXfvJbg= -github.com/pingcap/kvproto v0.0.0-20191104103048-40f562012fb1/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20191113075618-7ce83b774d70 h1:l9VcGUPRHvmM7mkFHo4JqxZeCvioRuL1/4tFUQcs6jQ= +github.com/pingcap/kvproto v0.0.0-20191113075618-7ce83b774d70/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20191113115126-45e0702fff1e/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20191118030148-ec389ef1b41f h1:CJ1IdT7bPbIvyq2Of9VhC/fhEGh6+0ItdT1dPBv7x7I= +github.com/pingcap/kvproto v0.0.0-20191118030148-ec389ef1b41f/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20191118050206-47672e7eabc0 h1:CHOC95Ct4abJ9EdmWqzpUxV+bgjB4lOxd3AFxqgoyzQ= +github.com/pingcap/kvproto v0.0.0-20191118050206-47672e7eabc0/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= diff --git a/store/mockstore/mocktikv/mock_tikv_test.go b/store/mockstore/mocktikv/mock_tikv_test.go index fa07665438a6a..7b1904e531959 100644 --- a/store/mockstore/mocktikv/mock_tikv_test.go +++ b/store/mockstore/mocktikv/mock_tikv_test.go @@ -661,40 +661,52 @@ func (s *testMVCCLevelDB) TestErrors(c *C) { } func (s *testMVCCLevelDB) TestCheckTxnStatus(c *C) { - s.mustPrewriteWithTTLOK(c, putMutations("pk", "val"), "pk", 5, 666) + startTS := uint64(5 << 18) + s.mustPrewriteWithTTLOK(c, putMutations("pk", "val"), "pk", startTS, 666) - ttl, commitTS, err := s.store.CheckTxnStatus([]byte("pk"), 5, 0, 666, false) + ttl, commitTS, action, err := s.store.CheckTxnStatus([]byte("pk"), startTS, startTS+100, 666, false) c.Assert(err, IsNil) c.Assert(ttl, Equals, uint64(666)) c.Assert(commitTS, Equals, uint64(0)) + c.Assert(action, Equals, kvrpcpb.Action_MinCommitTSPushed) - s.mustCommitOK(c, [][]byte{[]byte("pk")}, 5, 30) + s.mustCommitOK(c, [][]byte{[]byte("pk")}, startTS, startTS+101) - ttl, commitTS, err = s.store.CheckTxnStatus([]byte("pk"), 5, 0, 666, false) + ttl, commitTS, _, err = s.store.CheckTxnStatus([]byte("pk"), startTS, 0, 666, false) c.Assert(err, IsNil) c.Assert(ttl, Equals, uint64(0)) - c.Assert(commitTS, Equals, uint64(30)) + c.Assert(commitTS, Equals, uint64(startTS+101)) + + s.mustPrewriteWithTTLOK(c, putMutations("pk1", "val"), "pk1", startTS, 666) + s.mustRollbackOK(c, [][]byte{[]byte("pk1")}, startTS) - s.mustPrewriteWithTTLOK(c, putMutations("pk1", "val"), "pk1", 5, 666) - s.mustRollbackOK(c, [][]byte{[]byte("pk1")}, 5) + ttl, commitTS, action, err = s.store.CheckTxnStatus([]byte("pk1"), startTS, 0, 666, false) + c.Assert(err, IsNil) + c.Assert(ttl, Equals, uint64(0)) + c.Assert(commitTS, Equals, uint64(0)) + c.Assert(action, Equals, kvrpcpb.Action_NoAction) - ttl, commitTS, err = s.store.CheckTxnStatus([]byte("pk1"), 5, 0, 666, false) + s.mustPrewriteWithTTLOK(c, putMutations("pk2", "val"), "pk2", startTS, 666) + currentTS := uint64(777 << 18) + ttl, commitTS, action, err = s.store.CheckTxnStatus([]byte("pk2"), startTS, 0, currentTS, false) c.Assert(err, IsNil) c.Assert(ttl, Equals, uint64(0)) c.Assert(commitTS, Equals, uint64(0)) + c.Assert(action, Equals, kvrpcpb.Action_TTLExpireRollback) // Cover the TxnNotFound case. - _, _, err = s.store.CheckTxnStatus([]byte("txnNotFound"), 5, 0, 666, false) + _, _, _, err = s.store.CheckTxnStatus([]byte("txnNotFound"), 5, 0, 666, false) c.Assert(err, NotNil) notFound, ok := errors.Cause(err).(*ErrTxnNotFound) c.Assert(ok, IsTrue) c.Assert(notFound.StartTs, Equals, uint64(5)) c.Assert(string(notFound.PrimaryKey), Equals, "txnNotFound") - ttl, commitTS, err = s.store.CheckTxnStatus([]byte("txnNotFound"), 5, 0, 666, true) + ttl, commitTS, action, err = s.store.CheckTxnStatus([]byte("txnNotFound"), 5, 0, 666, true) c.Assert(err, IsNil) c.Assert(ttl, Equals, uint64(0)) c.Assert(commitTS, Equals, uint64(0)) + c.Assert(action, Equals, kvrpcpb.Action_LockNotExistRollback) // Check the rollback tombstone blocks this prewrite which comes with a smaller startTS. req := &kvrpcpb.PrewriteRequest{ @@ -710,7 +722,7 @@ func (s *testMVCCLevelDB) TestCheckTxnStatus(c *C) { func (s *testMVCCLevelDB) TestRejectCommitTS(c *C) { s.mustPrewriteOK(c, putMutations("x", "A"), "x", 5) // Push the minCommitTS - _, _, err := s.store.CheckTxnStatus([]byte("x"), 5, 100, 100, false) + _, _, _, err := s.store.CheckTxnStatus([]byte("x"), 5, 100, 100, false) c.Assert(err, IsNil) err = s.store.Commit([][]byte{[]byte("x")}, 5, 10) e, ok := errors.Cause(err).(*ErrCommitTSExpired) diff --git a/store/mockstore/mocktikv/mvcc.go b/store/mockstore/mocktikv/mvcc.go index d5067adbdc1dd..759f7a7f12399 100644 --- a/store/mockstore/mocktikv/mvcc.go +++ b/store/mockstore/mocktikv/mvcc.go @@ -268,7 +268,7 @@ type MVCCStore interface { BatchResolveLock(startKey, endKey []byte, txnInfos map[uint64]uint64) error GC(startKey, endKey []byte, safePoint uint64) error DeleteRange(startKey, endKey []byte) error - CheckTxnStatus(primaryKey []byte, lockTS uint64, startTS, currentTS uint64, rollbackIfNotFound bool) (ttl, commitTS uint64, err error) + CheckTxnStatus(primaryKey []byte, lockTS uint64, startTS, currentTS uint64, rollbackIfNotFound bool) (uint64, uint64, kvrpcpb.Action, error) Close() error } diff --git a/store/mockstore/mocktikv/mvcc_leveldb.go b/store/mockstore/mocktikv/mvcc_leveldb.go index 7f8349dd57b02..2e6dd599b945d 100644 --- a/store/mockstore/mocktikv/mvcc_leveldb.go +++ b/store/mockstore/mocktikv/mvcc_leveldb.go @@ -1032,10 +1032,13 @@ func (mvcc *MVCCLevelDB) Cleanup(key []byte, startTS, currentTS uint64) error { // primaryKey + lockTS together could locate the primary lock. // callerStartTS is the start ts of reader transaction. // currentTS is the current ts, but it may be inaccurate. Just use it to check TTL. -func (mvcc *MVCCLevelDB) CheckTxnStatus(primaryKey []byte, lockTS, callerStartTS, currentTS uint64, rollbackIfNotExist bool) (uint64, uint64, error) { +func (mvcc *MVCCLevelDB) CheckTxnStatus(primaryKey []byte, lockTS, callerStartTS, currentTS uint64, + rollbackIfNotExist bool) (ttl uint64, commitTS uint64, action kvrpcpb.Action, err error) { mvcc.mu.Lock() defer mvcc.mu.Unlock() + action = kvrpcpb.Action_NoAction + startKey := mvccEncode(primaryKey, lockVer) iter := newIterator(mvcc.db, &util.Range{ Start: startKey, @@ -1046,9 +1049,11 @@ func (mvcc *MVCCLevelDB) CheckTxnStatus(primaryKey []byte, lockTS, callerStartTS dec := lockDecoder{ expectKey: primaryKey, } - ok, err := dec.Decode(iter) + var ok bool + ok, err = dec.Decode(iter) if err != nil { - return 0, 0, errors.Trace(err) + err = errors.Trace(err) + return } // If current transaction's lock exists. if ok && dec.lock.startTS == lockTS { @@ -1058,17 +1063,20 @@ func (mvcc *MVCCLevelDB) CheckTxnStatus(primaryKey []byte, lockTS, callerStartTS // If the lock has already outdated, clean up it. if uint64(oracle.ExtractPhysical(lock.startTS))+lock.ttl < uint64(oracle.ExtractPhysical(currentTS)) { if err = rollbackLock(batch, primaryKey, lockTS); err != nil { - return 0, 0, errors.Trace(err) + err = errors.Trace(err) + return } if err = mvcc.db.Write(batch, nil); err != nil { - return 0, 0, errors.Trace(err) + err = errors.Trace(err) + return } - return 0, 0, nil + return 0, 0, kvrpcpb.Action_TTLExpireRollback, nil } // If this is a large transaction and the lock is active, push forward the minCommitTS. - // lock.minCommitTS == 0 may be a secondary lock, or not a large transaction. + // lock.minCommitTS == 0 may be a secondary lock, or not a large transaction (old version TiDB). if lock.minCommitTS > 0 { + action = kvrpcpb.Action_MinCommitTSPushed // We *must* guarantee the invariance lock.minCommitTS >= callerStartTS + 1 if lock.minCommitTS < callerStartTS+1 { lock.minCommitTS = callerStartTS + 1 @@ -1081,33 +1089,36 @@ func (mvcc *MVCCLevelDB) CheckTxnStatus(primaryKey []byte, lockTS, callerStartTS } writeKey := mvccEncode(primaryKey, lockVer) - writeValue, err := lock.MarshalBinary() - if err != nil { - return 0, 0, errors.Trace(err) + writeValue, err1 := lock.MarshalBinary() + if err1 != nil { + err = errors.Trace(err1) + return } batch.Put(writeKey, writeValue) - if err = mvcc.db.Write(batch, nil); err != nil { - return 0, 0, errors.Trace(err) + if err1 = mvcc.db.Write(batch, nil); err1 != nil { + err = errors.Trace(err1) + return } } } - return lock.ttl, 0, nil + return lock.ttl, 0, action, nil } // If current transaction's lock does not exist. // If the commit info of the current transaction exists. - c, ok, err := getTxnCommitInfo(iter, primaryKey, lockTS) - if err != nil { - return 0, 0, errors.Trace(err) + c, ok, err1 := getTxnCommitInfo(iter, primaryKey, lockTS) + if err1 != nil { + err = errors.Trace(err1) + return } if ok { // If current transaction is already committed. if c.valueType != typeRollback { - return 0, c.commitTS, nil + return 0, c.commitTS, action, nil } // If current transaction is already rollback. - return 0, 0, nil + return 0, 0, kvrpcpb.Action_NoAction, nil } } @@ -1120,16 +1131,18 @@ func (mvcc *MVCCLevelDB) CheckTxnStatus(primaryKey []byte, lockTS, callerStartTS if rollbackIfNotExist { batch := &leveldb.Batch{} - if err := rollbackLock(batch, primaryKey, lockTS); err != nil { - return 0, 0, errors.Trace(err) + if err1 := rollbackLock(batch, primaryKey, lockTS); err1 != nil { + err = errors.Trace(err1) + return } - if err := mvcc.db.Write(batch, nil); err != nil { - return 0, 0, errors.Trace(err) + if err1 := mvcc.db.Write(batch, nil); err1 != nil { + err = errors.Trace(err1) + return } - return 0, 0, nil + return 0, 0, kvrpcpb.Action_LockNotExistRollback, nil } - return 0, 0, &ErrTxnNotFound{kvrpcpb.TxnNotFound{ + return 0, 0, action, &ErrTxnNotFound{kvrpcpb.TxnNotFound{ StartTs: lockTS, PrimaryKey: primaryKey, }} diff --git a/store/mockstore/mocktikv/rpc.go b/store/mockstore/mocktikv/rpc.go index c87fb76b23cad..f10fef665c388 100644 --- a/store/mockstore/mocktikv/rpc.go +++ b/store/mockstore/mocktikv/rpc.go @@ -387,11 +387,11 @@ func (h *rpcHandler) handleKvCheckTxnStatus(req *kvrpcpb.CheckTxnStatusRequest) panic("KvCheckTxnStatus: key not in region") } var resp kvrpcpb.CheckTxnStatusResponse - ttl, commitTS, err := h.mvccStore.CheckTxnStatus(req.GetPrimaryKey(), req.GetLockTs(), req.GetCallerStartTs(), req.GetCurrentTs(), req.GetRollbackIfNotExist()) + ttl, commitTS, action, err := h.mvccStore.CheckTxnStatus(req.GetPrimaryKey(), req.GetLockTs(), req.GetCallerStartTs(), req.GetCurrentTs(), req.GetRollbackIfNotExist()) if err != nil { resp.Error = convertToKeyError(err) } else { - resp.LockTtl, resp.CommitVersion = ttl, commitTS + resp.LockTtl, resp.CommitVersion, resp.Action = ttl, commitTS, action } return &resp } diff --git a/store/tikv/lock_resolver.go b/store/tikv/lock_resolver.go index 27cfbd398b0e2..dc09d0c5bcaa0 100644 --- a/store/tikv/lock_resolver.go +++ b/store/tikv/lock_resolver.go @@ -109,6 +109,7 @@ func NewLockResolver(etcdAddrs []string, security config.Security) (*LockResolve type TxnStatus struct { ttl uint64 commitTS uint64 + action kvrpcpb.Action } // IsCommitted returns true if the txn's final status is Commit. @@ -397,7 +398,7 @@ func (lr *LockResolver) getTxnStatusFromLock(bo *Backoffer, l *Lock, callerStart } if l.LockType == kvrpcpb.Op_PessimisticLock { - return TxnStatus{l.TTL, 0}, nil + return TxnStatus{ttl: l.TTL}, nil } // Handle txnNotFound error. @@ -482,6 +483,7 @@ func (lr *LockResolver) getTxnStatus(bo *Backoffer, txnID uint64, primary []byte logutil.BgLogger().Error("getTxnStatus error", zap.Error(err)) return status, err } + status.action = cmdResp.Action if cmdResp.LockTtl != 0 { status.ttl = cmdResp.LockTtl } else { diff --git a/store/tikv/lock_test.go b/store/tikv/lock_test.go index 245695272b77e..c998f860adee9 100644 --- a/store/tikv/lock_test.go +++ b/store/tikv/lock_test.go @@ -201,7 +201,7 @@ func (s *testLockSuite) TestGetTxnStatus(c *C) { status, err = s.store.lockResolver.GetTxnStatus(startTS, startTS, []byte("a")) c.Assert(err, IsNil) c.Assert(status.IsCommitted(), IsFalse) - c.Assert(status.ttl, Greater, uint64(0)) + c.Assert(status.ttl, Greater, uint64(0), Commentf("action:%s", status.action)) } func (s *testLockSuite) TestCheckTxnStatusTTL(c *C) { @@ -234,6 +234,7 @@ func (s *testLockSuite) TestCheckTxnStatusTTL(c *C) { c.Assert(err, IsNil) c.Assert(status.ttl, Equals, uint64(0)) c.Assert(status.commitTS, Equals, uint64(0)) + c.Assert(status.action, Equals, kvrpcpb.Action_NoAction) // Check a committed txn. startTS, commitTS := s.putKV(c, []byte("a"), []byte("a")) @@ -279,6 +280,8 @@ func (s *testLockSuite) TestCheckTxnStatus(c *C) { oracle := s.store.GetOracle() currentTS, err := oracle.GetTimestamp(context.Background()) c.Assert(err, IsNil) + c.Assert(currentTS, Greater, txn.StartTS()) + bo := NewBackoffer(context.Background(), PrewriteMaxBackoff) resolver := newLockResolver(s.store) // Call getTxnStatus to check the lock status. @@ -287,6 +290,9 @@ func (s *testLockSuite) TestCheckTxnStatus(c *C) { c.Assert(status.IsCommitted(), IsFalse) c.Assert(status.ttl, Greater, uint64(0)) c.Assert(status.CommitTS(), Equals, uint64(0)) + // TODO: It should be Action_MinCommitTSPushed if minCommitTS is set in the Prewrite request. + // Update here to kvrpcpb.Action_MinCommitTSPushed in the next PR. + c.Assert(status.action, Equals, kvrpcpb.Action_NoAction) // Test the ResolveLocks API lock := s.mustGetLock(c, []byte("second")) @@ -303,10 +309,11 @@ func (s *testLockSuite) TestCheckTxnStatus(c *C) { // Then call getTxnStatus again and check the lock status. currentTS, err = oracle.GetTimestamp(context.Background()) c.Assert(err, IsNil) - status, err = resolver.getTxnStatus(bo, txn.StartTS(), []byte("key"), currentTS, currentTS, true) + status, err = newLockResolver(s.store).getTxnStatus(bo, txn.StartTS(), []byte("key"), currentTS, 0, true) c.Assert(err, IsNil) c.Assert(status.ttl, Equals, uint64(0)) c.Assert(status.commitTS, Equals, uint64(0)) + c.Assert(status.action, Equals, kvrpcpb.Action_NoAction) // Call getTxnStatus on a committed transaction. startTS, commitTS := s.putKV(c, []byte("a"), []byte("a")) @@ -366,7 +373,7 @@ func (s *testLockSuite) TestCheckTxnStatusNoWait(c *C) { c.Assert(err, IsNil) lock = &Lock{ Key: []byte("second"), - Primary: []byte("key"), + Primary: []byte("key_not_exist"), TxnID: startTS, TTL: 1000, } @@ -374,6 +381,7 @@ func (s *testLockSuite) TestCheckTxnStatusNoWait(c *C) { c.Assert(err, IsNil) c.Assert(status.ttl, Equals, uint64(0)) c.Assert(status.commitTS, Equals, uint64(0)) + c.Assert(status.action, Equals, kvrpcpb.Action_LockNotExistRollback) } func (s *testLockSuite) prewriteTxn(c *C, txn *tikvTxn) { From a99672c23478485b79bf18265c8d3ca4de018290 Mon Sep 17 00:00:00 2001 From: CWang Date: Mon, 18 Nov 2019 18:24:13 +0800 Subject: [PATCH 13/14] expression:implement vectorized evaluation for builtinInet6NtoaSig (#13504) --- expression/builtin_miscellaneous_vec.go | 31 ++++++++++++++++++-- expression/builtin_miscellaneous_vec_test.go | 14 +++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/expression/builtin_miscellaneous_vec.go b/expression/builtin_miscellaneous_vec.go index d60aa1ce4e279..266033fab53a6 100644 --- a/expression/builtin_miscellaneous_vec.go +++ b/expression/builtin_miscellaneous_vec.go @@ -16,6 +16,7 @@ package expression import ( "bytes" "encoding/binary" + "fmt" "math" "net" "strings" @@ -484,11 +485,37 @@ func (b *builtinInetAtonSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column } func (b *builtinInet6NtoaSig) vectorized() bool { - return false + return true } func (b *builtinInet6NtoaSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + val, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(val) + if err := b.args[0].VecEvalString(b.ctx, input, val); err != nil { + return err + } + result.ReserveString(n) + for i := 0; i < n; i++ { + if val.IsNull(i) { + result.AppendNull() + continue + } + valI := val.GetString(i) + ip := net.IP([]byte(valI)).String() + if len(valI) == net.IPv6len && !strings.Contains(ip, ":") { + ip = fmt.Sprintf("::ffff:%s", ip) + } + if net.ParseIP(ip) == nil { + result.AppendNull() + continue + } + result.AppendString(ip) + } + return nil } func (b *builtinNameConstRealSig) vectorized() bool { diff --git a/expression/builtin_miscellaneous_vec_test.go b/expression/builtin_miscellaneous_vec_test.go index 841bacc3e63a4..9de7761cb388a 100644 --- a/expression/builtin_miscellaneous_vec_test.go +++ b/expression/builtin_miscellaneous_vec_test.go @@ -28,9 +28,17 @@ var vecBuiltinMiscellaneousCases = map[string][]vecExprBenchCase{ ast.IsIPv6: { {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETString}}, }, - ast.Sleep: {}, - ast.UUID: {}, - ast.Inet6Ntoa: {}, + ast.Sleep: {}, + ast.UUID: {}, + ast.Inet6Ntoa: { + {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString}, geners: []dataGenerator{ + &selectStringGener{ + candidates: []string{ + "192.168.0.1", + "2001:db8::68", //ipv6 + }, + }}}, + }, ast.InetAton: { {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETString}, geners: []dataGenerator{&ipv4StrGener{}}}, {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETString}}, From 8558c7f0a3d17a90b38fd138147365e6504b5e6b Mon Sep 17 00:00:00 2001 From: Wenxuan Date: Mon, 18 Nov 2019 18:32:23 +0800 Subject: [PATCH 14/14] Revert "expression: implement vectorized evaluation for `builtinFormatSig` (#13215)" (#13545) --- expression/builtin_string_vec.go | 84 +-------------------------- expression/builtin_string_vec_test.go | 4 -- 2 files changed, 2 insertions(+), 86 deletions(-) diff --git a/expression/builtin_string_vec.go b/expression/builtin_string_vec.go index a4ba894f7a258..6404eb9e18409 100644 --- a/expression/builtin_string_vec.go +++ b/expression/builtin_string_vec.go @@ -2199,91 +2199,11 @@ func (b *builtinBinSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) } func (b *builtinFormatSig) vectorized() bool { - return true -} - -// vecEvalNumDecArgsForFormat evaluates first 2 arguments, i.e, x and d, for function `format`. -func vecEvalNumDecArgsForFormat(b *baseBuiltinFunc, input *chunk.Chunk) ([]string, []string, []bool, error) { - n := input.NumRows() - isDecimal := b.args[0].GetType().EvalType() == types.ETDecimal - var buf0 *chunk.Column - buf0, err := b.bufAllocator.get(b.args[0].GetType().EvalType(), n) - if err != nil { - return nil, nil, nil, err - } - defer b.bufAllocator.put(buf0) - buf1, err := b.bufAllocator.get(types.ETInt, n) - if err != nil { - return nil, nil, nil, err - } - defer b.bufAllocator.put(buf1) - if err := b.args[1].VecEvalInt(b.ctx, input, buf1); err != nil { - return nil, nil, nil, err - } - - xStrBuf := make([]string, n) - if isDecimal { - if err := b.args[0].VecEvalDecimal(b.ctx, input, buf0); err != nil { - return nil, nil, nil, err - } - buf1.MergeNulls(buf0) - d64s := buf0.Decimals() - for i := 0; i < n; i++ { - if buf1.IsNull(i) { - continue - } - xStrBuf[i] = d64s[i].String() - } - } else { - if err := b.args[0].VecEvalReal(b.ctx, input, buf0); err != nil { - return nil, nil, nil, err - } - buf1.MergeNulls(buf0) - f64s := buf0.Float64s() - for i := 0; i < n; i++ { - if buf1.IsNull(i) { - continue - } - xStrBuf[i] = fmt.Sprintf(strconv.FormatFloat(f64s[i], 'f', -1, 64)) - } - } - - d64s := buf1.Int64s() - isNulls := make([]bool, n) - dStrBuf := make([]string, n) - for i := 0; i < n; i++ { - if buf1.IsNull(i) { - isNulls[i] = true - continue - } - isNulls[i] = false - dStrBuf[i] = strconv.FormatInt(d64s[i], 10) - } - return xStrBuf, dStrBuf, isNulls, nil + return false } -// evalString evals FORMAT(X,D). -// See https://dev.mysql.com/doc/refman/5.7/en/string-functions.html#function_format func (b *builtinFormatSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { - xStrBuf, dStrBuf, isNulls, err := vecEvalNumDecArgsForFormat(&b.baseBuiltinFunc, input) - if err != nil { - return err - } - n := input.NumRows() - result.ReserveString(n) - var formatString string - for i := 0; i < n; i++ { - if isNulls[i] { - result.AppendNull() - continue - } - formatString, err = mysql.GetLocaleFormatFunction("en_US")(xStrBuf[i], dStrBuf[i]) - if err != nil { - return err - } - result.AppendString(formatString) - } - return nil + return errors.Errorf("not implemented") } func (b *builtinRightBinarySig) vectorized() bool { diff --git a/expression/builtin_string_vec_test.go b/expression/builtin_string_vec_test.go index 6a0e930fc62ff..1cd2c4428989b 100644 --- a/expression/builtin_string_vec_test.go +++ b/expression/builtin_string_vec_test.go @@ -331,10 +331,6 @@ var vecBuiltinStringCases = map[string][]vecExprBenchCase{ }, }}, }, - ast.Format: { - {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETDecimal, types.ETInt}, geners: []dataGenerator{&rangeDecimalGener{}, &rangeInt64Gener{0, 30}}}, - {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETReal, types.ETInt}, geners: []dataGenerator{&rangeRealGener{}, &rangeInt64Gener{0, 30}}}, - }, } func (s *testEvaluatorSuite) TestVectorizedBuiltinStringEvalOneVec(c *C) {