diff --git a/expression/builtin_string.go b/expression/builtin_string.go index c494d9fcb5c10..acac019139708 100644 --- a/expression/builtin_string.go +++ b/expression/builtin_string.go @@ -1150,7 +1150,7 @@ func (b *builtinConvertSig) evalString(row chunk.Row) (string, bool, error) { return string(ret), false, err } enc := charset.FindEncoding(resultTp.Charset) - if !charset.IsValidString(enc, expr) { + if !enc.IsValid(hack.Slice(expr)) { replace, _ := enc.Transform(nil, hack.Slice(expr), charset.OpReplace) return string(replace), false, nil } diff --git a/expression/builtin_string_vec.go b/expression/builtin_string_vec.go index 3da555f9319ed..202a3d74ed3f1 100644 --- a/expression/builtin_string_vec.go +++ b/expression/builtin_string_vec.go @@ -689,7 +689,7 @@ func (b *builtinConvertSig) vecEvalString(input *chunk.Chunk, result *chunk.Colu continue } exprI := expr.GetBytes(i) - if !charset.IsValid(enc, exprI) { + if !enc.IsValid(exprI) { encBuf, _ = enc.Transform(encBuf, exprI, charset.OpReplace) result.AppendBytes(encBuf) } else { diff --git a/expression/collation.go b/expression/collation.go index 8dc5df02e55e0..813560775e2b4 100644 --- a/expression/collation.go +++ b/expression/collation.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/logutil" ) @@ -315,7 +316,7 @@ func safeConvert(ctx sessionctx.Context, ec *ExprCollation, args ...Expression) if isNull { continue } - if !charset.IsValidString(enc, str) { + if !enc.IsValid(hack.Slice(str)) { return false } } else { diff --git a/parser/charset/encoding.go b/parser/charset/encoding.go index 92b1433549860..bf3d6b8ff269c 100644 --- a/parser/charset/encoding.go +++ b/parser/charset/encoding.go @@ -57,7 +57,7 @@ type Encoding interface { Tp() EncodingTp // Peek returns the next char. Peek(src []byte) []byte - // IsValid checks whether a utf-8 string is valid in the current encoding. + // IsValid checks whether the utf-8 bytes can be convert to valid string in current encoding. IsValid(src []byte) bool // Foreach iterates the characters in in current encoding. Foreach(src []byte, op Op, fn func(from, to []byte, ok bool) bool) @@ -103,21 +103,6 @@ const ( OpDecodeReplace = opToUTF8 | opTruncateReplace | opCollectTo ) -// IsValid checks whether the bytes is valid in current encoding. -func IsValid(e Encoding, src []byte) bool { - isValid := true - e.Foreach(src, opFromUTF8, func(from, to []byte, ok bool) bool { - isValid = ok - return ok - }) - return isValid -} - -// IsValidString is a string version of IsValid. -func IsValidString(e Encoding, str string) bool { - return IsValid(e, Slice(str)) -} - // CountValidBytes counts the first valid bytes in src that // can be encode to the current encoding. func CountValidBytes(e Encoding, src []byte) int { diff --git a/parser/charset/encoding_test.go b/parser/charset/encoding_test.go index a78aa640d8be5..27d41dbf5ebd2 100644 --- a/parser/charset/encoding_test.go +++ b/parser/charset/encoding_test.go @@ -133,8 +133,7 @@ func TestEncodingValidate(t *testing.T) { enc = charset.EncodingUTF8MB3StrictImpl } strBytes := []byte(tc.str) - ok := charset.IsValid(enc, strBytes) - require.Equal(t, tc.ok, ok, msg) + require.Equal(t, tc.ok, enc.IsValid(strBytes), msg) replace, _ := enc.Transform(nil, strBytes, charset.OpReplace) require.Equal(t, tc.expected, string(replace), msg) } diff --git a/parser/charset/encoding_utf8.go b/parser/charset/encoding_utf8.go index 6c7d79941a31d..499ce5ea50de7 100644 --- a/parser/charset/encoding_utf8.go +++ b/parser/charset/encoding_utf8.go @@ -77,7 +77,7 @@ func (e *encodingUTF8) IsValid(src []byte) bool { // Transform implements Encoding interface. func (e *encodingUTF8) Transform(dest, src []byte, op Op) ([]byte, error) { - if IsValid(e, src) { + if e.IsValid(src) { return src, nil } return e.encodingBase.Transform(dest, src, op) @@ -120,7 +120,7 @@ func (e *encodingUTF8MB3Strict) Foreach(src []byte, op Op, fn func(srcCh, dstCh // Transform implements Encoding interface. func (e *encodingUTF8MB3Strict) Transform(dest, src []byte, op Op) ([]byte, error) { - if IsValid(e, src) { + if e.IsValid(src) { return src, nil } return e.encodingBase.Transform(dest, src, op)