From 88b36452911afc76ba8ec09a2f27c8775f06b63f Mon Sep 17 00:00:00 2001 From: Weijun-H Date: Mon, 24 Oct 2022 16:41:24 +0000 Subject: [PATCH 1/5] add support for CONV Signed-off-by: Weijun-H --- go/vt/vtgate/evalengine/func.go | 1 + .../evalengine/integration/string_fun_test.go | 32 ++++++++++++++++ go/vt/vtgate/evalengine/string.go | 37 +++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/go/vt/vtgate/evalengine/func.go b/go/vt/vtgate/evalengine/func.go index cbe530956a5..65c5e93f266 100644 --- a/go/vt/vtgate/evalengine/func.go +++ b/go/vt/vtgate/evalengine/func.go @@ -49,6 +49,7 @@ var builtinFunctions = map[string]builtin{ "bit_length": builtinBitLength{}, "ascii": builtinASCII{}, "repeat": builtinRepeat{}, + "conv": builtinConv{}, } var builtinFunctionsRewrite = map[string]builtinRewrite{ diff --git a/go/vt/vtgate/evalengine/integration/string_fun_test.go b/go/vt/vtgate/evalengine/integration/string_fun_test.go index af37ce03d24..e133bed92a5 100644 --- a/go/vt/vtgate/evalengine/integration/string_fun_test.go +++ b/go/vt/vtgate/evalengine/integration/string_fun_test.go @@ -204,3 +204,35 @@ func TestBuiltinRepeat(t *testing.T) { } } + +func TestBuiltinConv(t *testing.T) { + var conn = mysqlconn(t) + defer conn.Close() + cases := []string{ + "10", + "10 + '10' + 10", + "-10", + "'10'", + } + bases := []string{ + "-1", + "1", + "2", + "4", + "8", + "10", + "16", + "32", + } + + for _, num := range cases { + for _, fromBase := range bases { + for _, toBase := range bases { + query := fmt.Sprintf("CONV(%s, %s, %s)", num, fromBase, toBase) + compareRemoteExpr(t, conn, query) + } + } + + } + +} diff --git a/go/vt/vtgate/evalengine/string.go b/go/vt/vtgate/evalengine/string.go index 668ec11cf6d..c4493f07109 100644 --- a/go/vt/vtgate/evalengine/string.go +++ b/go/vt/vtgate/evalengine/string.go @@ -18,6 +18,9 @@ package evalengine import ( "bytes" + "fmt" + "strconv" + "strings" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" @@ -265,3 +268,37 @@ func (builtinRepeat) typeof(env *ExpressionEnv, args []Expr) (sqltypes.Type, fla return sqltypes.VarChar, f1 } + +type builtinConv struct { +} + +func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResult) { + inarg := &args[0] + fromBase := args[1] + toBase := args[2] + + if inarg.isNull() || fromBase.isNull() || toBase.isNull() || fromBase.int64() < 2 || fromBase.int64() > 36 || toBase.int64() < 2 || toBase.int64() > 36 { + result.setNull() + return + } + + inarg.makeSignedIntegral() + + num, _ := strconv.ParseInt(fmt.Sprint(inarg.int64()), int(fromBase.int64()), 64) + + convNum := strconv.FormatUint(uint64(num), int(toBase.int64())) + convNum = strings.ToUpper(convNum) + result.setRaw(sqltypes.VarChar, []byte(convNum), inarg.collation()) + result.makeTextual(env.DefaultCollation) +} + +func (builtinConv) typeof(env *ExpressionEnv, args []Expr) (sqltypes.Type, flag) { + if len(args) != 3 { + throwArgError("CONV") + } + _, f1 := args[0].typeof(env) + // typecheck the right-hand argument but ignore its flags + args[1].typeof(env) + + return sqltypes.VarChar, f1 +} From f1654a395d6f04ebf3fe3d8da2fcc447b12c435f Mon Sep 17 00:00:00 2001 From: Weijun-H Date: Tue, 1 Nov 2022 17:24:32 +0000 Subject: [PATCH 2/5] failed to test hex literal Signed-off-by: Weijun-H --- .../evalengine/integration/string_fun_test.go | 17 ++++- go/vt/vtgate/evalengine/string.go | 76 ++++++++++++++++--- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/go/vt/vtgate/evalengine/integration/string_fun_test.go b/go/vt/vtgate/evalengine/integration/string_fun_test.go index e133bed92a5..2804fec06b2 100644 --- a/go/vt/vtgate/evalengine/integration/string_fun_test.go +++ b/go/vt/vtgate/evalengine/integration/string_fun_test.go @@ -209,10 +209,19 @@ func TestBuiltinConv(t *testing.T) { var conn = mysqlconn(t) defer conn.Close() cases := []string{ + "-5.1", + "-5.9", + "-0xa21 + '1'", "10", "10 + '10' + 10", + "10 + '10' - 10", "-10", "'10'", + "10+'10'+'10a'+X'0a'", + "10 / 10", + "X'0FFFFFFFFFFFFFF'", + "99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", + "-99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", } bases := []string{ "-1", @@ -223,13 +232,19 @@ func TestBuiltinConv(t *testing.T) { "10", "16", "32", + "64", } for _, num := range cases { for _, fromBase := range bases { - for _, toBase := range bases { + for i := range bases { + toBase := bases[i] query := fmt.Sprintf("CONV(%s, %s, %s)", num, fromBase, toBase) compareRemoteExpr(t, conn, query) + + toBase = bases[len(bases)-1-i] + query = fmt.Sprintf("CONV(%s, %s, %s)", num, fromBase, toBase) + compareRemoteExpr(t, conn, query) } } diff --git a/go/vt/vtgate/evalengine/string.go b/go/vt/vtgate/evalengine/string.go index c4493f07109..7e649938a71 100644 --- a/go/vt/vtgate/evalengine/string.go +++ b/go/vt/vtgate/evalengine/string.go @@ -19,6 +19,8 @@ package evalengine import ( "bytes" "fmt" + "math" + "regexp" "strconv" "strings" @@ -273,23 +275,75 @@ type builtinConv struct { } func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResult) { + const MaxUint = 18446744073709551615 + var fromNum uint64 + var fromNumNeg uint64 + var isNeg bool inarg := &args[0] - fromBase := args[1] - toBase := args[2] + inarg2 := &args[1] + inarg3 := &args[2] + fromBase := inarg2.int64() + toBase := inarg3.int64() + fromNum = 0 - if inarg.isNull() || fromBase.isNull() || toBase.isNull() || fromBase.int64() < 2 || fromBase.int64() > 36 || toBase.int64() < 2 || toBase.int64() > 36 { + if inarg.isNull() || inarg2.isNull() || inarg3.isNull() || fromBase < 2 || fromBase > 36 || toBase < 2 || toBase > 36 { result.setNull() return } - inarg.makeSignedIntegral() + rawString := string(inarg.toRawBytes()) + re, _ := regexp.Compile(`[+-]?[0-9.x]+[a-vA-Vx]*`) + for _, num := range re.FindAllString(rawString, -1) { + isNeg = false + reFindDot, _ := regexp.Compile(`\.`) + reFindNeg, _ := regexp.Compile(`^-[0-9.]+[a-vA-V]*`) + if reFindDot.MatchString(num) { + temp, _ := strconv.ParseFloat(num, 64) + temp = math.Trunc(temp) + num = fmt.Sprint(int64(temp)) + } + if reFindNeg.MatchString(num) { + isNeg = true + num = num[1:] + } + + if transNum, err := strconv.ParseUint(num, int(fromBase), 64); err == nil { + if isNeg { + fromNumNeg = fromNumNeg + transNum + } else { + fromNum = fromNum + transNum + } + } else if strings.Contains(err.Error(), "value out of range") { + if isNeg { + fromNumNeg = MaxUint + + } else { + fromNum = MaxUint + } + } else { + fromNum = 0 + break + } + } - num, _ := strconv.ParseInt(fmt.Sprint(inarg.int64()), int(fromBase.int64()), 64) + if fromNumNeg < fromNum { + fromNum = fromNum - fromNumNeg + } else if fromNumNeg == MaxUint { + fromNum = 0 + } else { + fromNum = fromNumNeg - fromNum + } + var toNum string + if isNeg { + temp := strconv.FormatUint(uint64(-fromNum), int(toBase)) + toNum = strings.ToUpper(temp) + } else { + temp := strconv.FormatUint(fromNum, int(toBase)) + toNum = strings.ToUpper(temp) + } - convNum := strconv.FormatUint(uint64(num), int(toBase.int64())) - convNum = strings.ToUpper(convNum) - result.setRaw(sqltypes.VarChar, []byte(convNum), inarg.collation()) - result.makeTextual(env.DefaultCollation) + inarg.makeTextualAndConvert(env.DefaultCollation) + result.setString(toNum, inarg.collation()) } func (builtinConv) typeof(env *ExpressionEnv, args []Expr) (sqltypes.Type, flag) { @@ -297,8 +351,10 @@ func (builtinConv) typeof(env *ExpressionEnv, args []Expr) (sqltypes.Type, flag) throwArgError("CONV") } _, f1 := args[0].typeof(env) + _, f2 := args[1].typeof(env) // typecheck the right-hand argument but ignore its flags args[1].typeof(env) + args[2].typeof(env) - return sqltypes.VarChar, f1 + return sqltypes.VarChar, f1 & f2 } From efbaff60ee7fa202267237696f26fd96195fe14b Mon Sep 17 00:00:00 2001 From: Weijun-H Date: Wed, 2 Nov 2022 18:25:35 +0000 Subject: [PATCH 3/5] support hex literal Signed-off-by: Weijun-H --- .../evalengine/integration/string_fun_test.go | 1 + go/vt/vtgate/evalengine/string.go | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/go/vt/vtgate/evalengine/integration/string_fun_test.go b/go/vt/vtgate/evalengine/integration/string_fun_test.go index 2804fec06b2..a3d44bb6648 100644 --- a/go/vt/vtgate/evalengine/integration/string_fun_test.go +++ b/go/vt/vtgate/evalengine/integration/string_fun_test.go @@ -211,6 +211,7 @@ func TestBuiltinConv(t *testing.T) { cases := []string{ "-5.1", "-5.9", + "0xa21 + '1'", "-0xa21 + '1'", "10", "10 + '10' + 10", diff --git a/go/vt/vtgate/evalengine/string.go b/go/vt/vtgate/evalengine/string.go index 7e649938a71..3e1f70e70e7 100644 --- a/go/vt/vtgate/evalengine/string.go +++ b/go/vt/vtgate/evalengine/string.go @@ -279,19 +279,37 @@ func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResul var fromNum uint64 var fromNumNeg uint64 var isNeg bool + var rawString string inarg := &args[0] inarg2 := &args[1] inarg3 := &args[2] fromBase := inarg2.int64() toBase := inarg3.int64() fromNum = 0 + t := inarg.typeof() if inarg.isNull() || inarg2.isNull() || inarg3.isNull() || fromBase < 2 || fromBase > 36 || toBase < 2 || toBase > 36 { result.setNull() return } - rawString := string(inarg.toRawBytes()) + rawString = string(inarg.toRawBytes()) + rawString = strings.ToLower(rawString) + + if t == sqltypes.Float64 { + for i, c := range rawString { + if c == '-' { + continue + } + if (fromBase <= 9 && c >= '0' && c <= rune('0'+fromBase)) || (fromBase > 9 && ((c >= '0' && c <= '9') || (c >= 'a' && c <= rune('a'+fromBase-9)))) { + continue + } else { + rawString = rawString[:i] + break + } + } + } + re, _ := regexp.Compile(`[+-]?[0-9.x]+[a-vA-Vx]*`) for _, num := range re.FindAllString(rawString, -1) { isNeg = false From 67575f3f1808eac3dcce45d27a399c3087c520a5 Mon Sep 17 00:00:00 2001 From: Weijun-H Date: Thu, 3 Nov 2022 15:24:01 +0000 Subject: [PATCH 4/5] support more tests Signed-off-by: Weijun-H --- .../evalengine/integration/string_fun_test.go | 8 +- go/vt/vtgate/evalengine/string.go | 80 +++++++++---------- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/go/vt/vtgate/evalengine/integration/string_fun_test.go b/go/vt/vtgate/evalengine/integration/string_fun_test.go index a3d44bb6648..86cd26c8ee6 100644 --- a/go/vt/vtgate/evalengine/integration/string_fun_test.go +++ b/go/vt/vtgate/evalengine/integration/string_fun_test.go @@ -209,11 +209,16 @@ func TestBuiltinConv(t *testing.T) { var conn = mysqlconn(t) defer conn.Close() cases := []string{ + "++5", + "--4", "-5.1", "-5.9", "0xa21 + '1'", + "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", "-0xa21 + '1'", - "10", + "'+10'", + "'10-9+10'", + "'+10-9+10'", "10 + '10' + 10", "10 + '10' - 10", "-10", @@ -234,6 +239,7 @@ func TestBuiltinConv(t *testing.T) { "16", "32", "64", + "0xa", } for _, num := range cases { diff --git a/go/vt/vtgate/evalengine/string.go b/go/vt/vtgate/evalengine/string.go index 3e1f70e70e7..45a7a1a95b2 100644 --- a/go/vt/vtgate/evalengine/string.go +++ b/go/vt/vtgate/evalengine/string.go @@ -18,9 +18,6 @@ package evalengine import ( "bytes" - "fmt" - "math" - "regexp" "strconv" "strings" @@ -283,12 +280,24 @@ func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResul inarg := &args[0] inarg2 := &args[1] inarg3 := &args[2] - fromBase := inarg2.int64() - toBase := inarg3.int64() + + if sqltypes.IsBinary(inarg.typeof()) { + inarg.makeUnsignedIntegral() + } + + if sqltypes.IsBinary(inarg2.typeof()) { + inarg2.makeUnsignedIntegral() + } + + if sqltypes.IsBinary(inarg3.typeof()) { + inarg3.makeUnsignedIntegral() + } + + fromBase, _ := strconv.Atoi(string(inarg2.toRawBytes())) + toBase, _ := strconv.Atoi(string(inarg3.toRawBytes())) fromNum = 0 - t := inarg.typeof() - if inarg.isNull() || inarg2.isNull() || inarg3.isNull() || fromBase < 2 || fromBase > 36 || toBase < 2 || toBase > 36 { + if inarg.isNull() || fromBase < 2 || fromBase > 36 || toBase < 2 || toBase > 36 { result.setNull() return } @@ -296,52 +305,41 @@ func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResul rawString = string(inarg.toRawBytes()) rawString = strings.ToLower(rawString) - if t == sqltypes.Float64 { - for i, c := range rawString { - if c == '-' { + trimStr := func(s string, isNeg *bool) string { + start := 0 + for i, c := range s { + if (c == '+' || c == '-') && i == 0 { + start++ + *isNeg = (c == '-') continue } - if (fromBase <= 9 && c >= '0' && c <= rune('0'+fromBase)) || (fromBase > 9 && ((c >= '0' && c <= '9') || (c >= 'a' && c <= rune('a'+fromBase-9)))) { + if (fromBase <= 9 && c >= '0' && c <= rune('0'+fromBase)) || + (fromBase > 9 && ((c >= '0' && c <= '9') || (c >= 'a' && c <= rune('a'+fromBase-9)))) { continue } else { - rawString = rawString[:i] - break + return s[start:i] } } + return s[start:] } - re, _ := regexp.Compile(`[+-]?[0-9.x]+[a-vA-Vx]*`) - for _, num := range re.FindAllString(rawString, -1) { - isNeg = false - reFindDot, _ := regexp.Compile(`\.`) - reFindNeg, _ := regexp.Compile(`^-[0-9.]+[a-vA-V]*`) - if reFindDot.MatchString(num) { - temp, _ := strconv.ParseFloat(num, 64) - temp = math.Trunc(temp) - num = fmt.Sprint(int64(temp)) - } - if reFindNeg.MatchString(num) { - isNeg = true - num = num[1:] - } + num := trimStr(rawString, &isNeg) - if transNum, err := strconv.ParseUint(num, int(fromBase), 64); err == nil { - if isNeg { - fromNumNeg = fromNumNeg + transNum - } else { - fromNum = fromNum + transNum - } - } else if strings.Contains(err.Error(), "value out of range") { - if isNeg { - fromNumNeg = MaxUint + if transNum, err := strconv.ParseUint(num, int(fromBase), 64); err == nil { + if isNeg { + fromNumNeg = fromNumNeg + transNum + } else { + fromNum = fromNum + transNum + } + } else if strings.Contains(err.Error(), "value out of range") { + if isNeg { + fromNumNeg = MaxUint - } else { - fromNum = MaxUint - } } else { - fromNum = 0 - break + fromNum = MaxUint } + } else { + fromNum = 0 } if fromNumNeg < fromNum { From 654abcb6bce9d64bf4da7e1e1eda9dd26167360e Mon Sep 17 00:00:00 2001 From: Weijun-H Date: Wed, 9 Nov 2022 15:24:53 +0000 Subject: [PATCH 5/5] support negetive base Signed-off-by: Weijun-H --- .../evalengine/integration/string_fun_test.go | 10 +++ go/vt/vtgate/evalengine/string.go | 78 ++++++++++++------- 2 files changed, 60 insertions(+), 28 deletions(-) diff --git a/go/vt/vtgate/evalengine/integration/string_fun_test.go b/go/vt/vtgate/evalengine/integration/string_fun_test.go index 86cd26c8ee6..e347632ff80 100644 --- a/go/vt/vtgate/evalengine/integration/string_fun_test.go +++ b/go/vt/vtgate/evalengine/integration/string_fun_test.go @@ -228,17 +228,27 @@ func TestBuiltinConv(t *testing.T) { "X'0FFFFFFFFFFFFFF'", "99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", "-99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", + "1000000000000000000000000000000", } bases := []string{ "-1", "1", "2", + "-2", "4", + "-4", "8", + "-8", "10", + "-10", "16", + "-16", "32", + "-32", "64", + "-64", + "128", + "-128", "0xa", } diff --git a/go/vt/vtgate/evalengine/string.go b/go/vt/vtgate/evalengine/string.go index 45a7a1a95b2..2e0bb577ec8 100644 --- a/go/vt/vtgate/evalengine/string.go +++ b/go/vt/vtgate/evalengine/string.go @@ -273,8 +273,9 @@ type builtinConv struct { func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResult) { const MaxUint = 18446744073709551615 + const MAXINT = 9223372036854775807 + const MININT = -9223372036854775808 var fromNum uint64 - var fromNumNeg uint64 var isNeg bool var rawString string inarg := &args[0] @@ -297,7 +298,9 @@ func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResul toBase, _ := strconv.Atoi(string(inarg3.toRawBytes())) fromNum = 0 - if inarg.isNull() || fromBase < 2 || fromBase > 36 || toBase < 2 || toBase > 36 { + if inarg.isNull() || + (fromBase > -2 && fromBase < 2) || (toBase > -2 && toBase < 2) || + fromBase < -36 || fromBase > 36 || toBase < -36 || toBase > 36 { result.setNull() return } @@ -306,6 +309,12 @@ func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResul rawString = strings.ToLower(rawString) trimStr := func(s string, isNeg *bool) string { + var base uint64 + if fromBase > 0 { + base = uint64(fromBase) + } else { + base = -uint64(fromBase) + } start := 0 for i, c := range s { if (c == '+' || c == '-') && i == 0 { @@ -313,8 +322,8 @@ func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResul *isNeg = (c == '-') continue } - if (fromBase <= 9 && c >= '0' && c <= rune('0'+fromBase)) || - (fromBase > 9 && ((c >= '0' && c <= '9') || (c >= 'a' && c <= rune('a'+fromBase-9)))) { + if (base <= 9 && c >= '0' && c <= rune('0'+base)) || + (base > 9 && ((c >= '0' && c <= '9') || (c >= 'a' && c <= rune('a'+base-9)))) { continue } else { return s[start:i] @@ -325,38 +334,52 @@ func (builtinConv) call(env *ExpressionEnv, args []EvalResult, result *EvalResul num := trimStr(rawString, &isNeg) - if transNum, err := strconv.ParseUint(num, int(fromBase), 64); err == nil { + if fromBase < 0 { if isNeg { - fromNumNeg = fromNumNeg + transNum - } else { - fromNum = fromNum + transNum + num = "-" + num } - } else if strings.Contains(err.Error(), "value out of range") { - if isNeg { - fromNumNeg = MaxUint - - } else { - fromNum = MaxUint + if transNum, err := strconv.ParseInt(num, -fromBase, 64); err == nil { + if isNeg { + fromNum = uint64(-transNum) + } else { + fromNum = uint64(transNum) + } + } else if strings.Contains(err.Error(), "value out of range") { + if isNeg { + fromNum = uint64(-MININT) + } else { + fromNum = uint64(MAXINT) + } } } else { - fromNum = 0 + if transNum, err := strconv.ParseUint(num, int(fromBase), 64); err == nil { + fromNum = transNum + } else if strings.Contains(err.Error(), "value out of range") { + if isNeg { + fromNum = 0 + } else { + fromNum = MaxUint + } + } } - if fromNumNeg < fromNum { - fromNum = fromNum - fromNumNeg - } else if fromNumNeg == MaxUint { - fromNum = 0 - } else { - fromNum = fromNumNeg - fromNum - } var toNum string - if isNeg { - temp := strconv.FormatUint(uint64(-fromNum), int(toBase)) - toNum = strings.ToUpper(temp) + var temp string + if toBase > 0 { + if isNeg { + temp = strconv.FormatUint(uint64(-fromNum), toBase) + } else { + temp = strconv.FormatUint(fromNum, toBase) + } } else { - temp := strconv.FormatUint(fromNum, int(toBase)) - toNum = strings.ToUpper(temp) + toBase = -toBase + if isNeg { + temp = strconv.FormatInt(int64(-fromNum), toBase) + } else { + temp = strconv.FormatInt(int64(fromNum), toBase) + } } + toNum = strings.ToUpper(temp) inarg.makeTextualAndConvert(env.DefaultCollation) result.setString(toNum, inarg.collation()) @@ -368,7 +391,6 @@ func (builtinConv) typeof(env *ExpressionEnv, args []Expr) (sqltypes.Type, flag) } _, f1 := args[0].typeof(env) _, f2 := args[1].typeof(env) - // typecheck the right-hand argument but ignore its flags args[1].typeof(env) args[2].typeof(env)