Merge branch 'master' into disable_direct_placement

cmd/explaintest/r/new_character_set.result

@@ -105,3 +105,9 @@ insert into t values('{"点赞": "你好"}', '{"点赞": "你好"}', '{"点赞":
 select * from t;
 j	b	s1	s2	st	en
 {"点赞": "你好"}	{"鐐硅禐": "浣犲ソ"}	{"鐐硅禐": "浣犲ソ"}	{"点赞": "你好"}	{"点赞": "你好"}	{"点赞": "你好"}
+set names utf8mb4;
+set @@character_set_client=gbk;
+set @@character_set_connection=gbk;
+select hex('一a'), '一a';
+hex('涓?')	涓?
+E4B83F	涓?

cmd/explaintest/t/new_character_set.test

@@ -63,3 +63,8 @@ set character_set_connection = utf8mb4;
 create table t(j json, b blob, s1 varchar(255) collate binary, s2 varchar(255), st set('{"点赞": "你好"}'), en enum('{"点赞": "你好"}'));
 insert into t values('{"点赞": "你好"}', '{"点赞": "你好"}', '{"点赞": "你好"}', '{"点赞": "你好"}', '{"点赞": "你好"}', '{"点赞": "你好"}');
 select * from t;
+
+set names utf8mb4;
+set @@character_set_client=gbk;
+set @@character_set_connection=gbk;
+select hex('一a'), '一a';

ddl/placement_policy_test.go

@@ -624,6 +624,124 @@ func (s *testDBSuite6) TestCreateTableWithPlacementPolicy(c *C) {
 	tk.MustExec("drop placement policy if exists y")
 }
 
+func (s *testDBSuite6) getClonedTable(dbName string, tableName string) (*model.TableInfo, error) {
+	tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr(dbName), model.NewCIStr(tableName))
+	if err != nil {
+		return nil, err
+	}
+
+	tblMeta := tbl.Meta()
+	tblMeta = tblMeta.Clone()
+	policyRef := *tblMeta.PlacementPolicyRef
+	tblMeta.PlacementPolicyRef = &policyRef
+	return tblMeta, nil
+}
+
+func (s *testDBSuite6) getClonedDatabase(dbName string) (*model.DBInfo, bool) {
+	db, ok := s.dom.InfoSchema().SchemaByName(model.NewCIStr(dbName))
+	if !ok {
+		return nil, ok
+	}
+
+	db = db.Clone()
+	policyRef := *db.PlacementPolicyRef
+	db.PlacementPolicyRef = &policyRef
+	return db, true
+}
+
+func (s *testDBSuite6) TestCreateTableWithInfoPlacement(c *C) {
+	clearAllBundles(c)
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t1")
+	tk.MustExec("drop database if exists test2")
+	tk.MustExec("drop placement policy if exists p1")
+
+	tk.MustExec("create placement policy p1 followers=1")
+	defer tk.MustExec("drop placement policy if exists p1")
+	tk.MustExec("create table t1(a int) placement policy p1")
+	defer tk.MustExec("drop table if exists t1")
+	tk.MustExec("create database test2")
+	defer tk.MustExec("drop database if exists test2")
+
+	tbl, err := s.getClonedTable("test", "t1")
+	c.Assert(err, IsNil)
+	policy, ok := s.dom.InfoSchema().PolicyByName(model.NewCIStr("p1"))
+	c.Assert(ok, IsTrue)
+	c.Assert(tbl.PlacementPolicyRef.ID, Equals, policy.ID)
+
+	tk.MustExec("alter table t1 placement policy='default'")
+	tk.MustExec("drop placement policy p1")
+	tk.MustExec("create placement policy p1 followers=2")
+	c.Assert(s.dom.DDL().CreateTableWithInfo(tk.Se, model.NewCIStr("test2"), tbl, ddl.OnExistError), IsNil)
+	tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" +
+		"  `a` int(11) DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+	tk.MustQuery("show create table test2.t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" +
+		"  `a` int(11) DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![placement] PLACEMENT POLICY=`p1` */"))
+	tk.MustQuery("show placement where target='TABLE test2.t1'").Check(testkit.Rows("TABLE test2.t1 FOLLOWERS=2 PENDING"))
+
+	// The ref id for new table should be the new policy id
+	tbl2, err := s.getClonedTable("test2", "t1")
+	c.Assert(err, IsNil)
+	policy2, ok := s.dom.InfoSchema().PolicyByName(model.NewCIStr("p1"))
+	c.Assert(ok, IsTrue)
+	c.Assert(tbl2.PlacementPolicyRef.ID, Equals, policy2.ID)
+	c.Assert(policy2.ID != policy.ID, IsTrue)
+
+	// Test policy not exists
+	tbl2.Name = model.NewCIStr("t3")
+	tbl2.PlacementPolicyRef.Name = model.NewCIStr("pxx")
+	err = s.dom.DDL().CreateTableWithInfo(tk.Se, model.NewCIStr("test2"), tbl2, ddl.OnExistError)
+	c.Assert(err.Error(), Equals, "[schema:8239]Unknown placement policy 'pxx'")
+}
+
+func (s *testDBSuite6) TestCreateSchemaWithInfoPlacement(c *C) {
+	clearAllBundles(c)
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop database if exists test2")
+	tk.MustExec("drop database if exists test3")
+	tk.MustExec("drop placement policy if exists p1")
+
+	tk.MustExec("create placement policy p1 followers=1")
+	defer tk.MustExec("drop placement policy if exists p1")
+	tk.MustExec("create database test2 placement policy p1")
+	defer tk.MustExec("drop database if exists test2")
+	defer tk.MustExec("drop database if exists test3")
+
+	db, ok := s.getClonedDatabase("test2")
+	c.Assert(ok, IsTrue)
+	policy, ok := s.dom.InfoSchema().PolicyByName(model.NewCIStr("p1"))
+	c.Assert(ok, IsTrue)
+	c.Assert(db.PlacementPolicyRef.ID, Equals, policy.ID)
+
+	db2 := db.Clone()
+	db2.Name = model.NewCIStr("test3")
+	tk.MustExec("alter database test2 placement policy='default'")
+	tk.MustExec("drop placement policy p1")
+	tk.MustExec("create placement policy p1 followers=2")
+	c.Assert(s.dom.DDL().CreateSchemaWithInfo(tk.Se, db2, ddl.OnExistError), IsNil)
+	tk.MustQuery("show create database test2").Check(testkit.Rows("test2 CREATE DATABASE `test2` /*!40100 DEFAULT CHARACTER SET utf8mb4 */"))
+	tk.MustQuery("show create database test3").Check(testkit.Rows("test3 CREATE DATABASE `test3` /*!40100 DEFAULT CHARACTER SET utf8mb4 */ /*T![placement] PLACEMENT POLICY=`p1` */"))
+	tk.MustQuery("show placement where target='DATABASE test3'").Check(testkit.Rows("DATABASE test3 FOLLOWERS=2 SCHEDULED"))
+
+	// The ref id for new table should be the new policy id
+	db2, ok = s.getClonedDatabase("test3")
+	c.Assert(ok, IsTrue)
+	policy2, ok := s.dom.InfoSchema().PolicyByName(model.NewCIStr("p1"))
+	c.Assert(ok, IsTrue)
+	c.Assert(db2.PlacementPolicyRef.ID, Equals, policy2.ID)
+	c.Assert(policy2.ID != policy.ID, IsTrue)
+
+	// Test policy not exists
+	db2.Name = model.NewCIStr("test4")
+	db2.PlacementPolicyRef.Name = model.NewCIStr("p2")
+	err := s.dom.DDL().CreateSchemaWithInfo(tk.Se, db2, ddl.OnExistError)
+	c.Assert(err.Error(), Equals, "[schema:8239]Unknown placement policy 'p2'")
+}
+
 func (s *testDBSuite6) TestDropPlacementPolicyInUse(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")

ddl/placement_sql_test.go

@@ -659,6 +659,53 @@ func (s *testDBSuite6) TestPlacementMode(c *C) {
 		" PARTITION `p1` VALUES LESS THAN (1000),\n" +
 		" PARTITION `p2` VALUES LESS THAN (10000),\n" +
 		" PARTITION `p3` VALUES LESS THAN (100000))"))
+
+	// create tableWithInfo in ignore mode
+	tk.MustExec("drop table if exists t2")
+	tbl, err := s.getClonedTable("test", "t1")
+	c.Assert(err, IsNil)
+	c.Assert(tbl.PlacementPolicyRef, NotNil)
+	tbl.Name = model.NewCIStr("t2")
+	err = s.dom.DDL().CreateTableWithInfo(tk.Se, model.NewCIStr("test"), tbl, ddl.OnExistError)
+	c.Assert(err, IsNil)
+	tk.MustQuery("show create table t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
+		"  `id` int(11) DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='aaa'"))
+
+	// createTableWithInfo in ignore mode (policy not exists)
+	tk.MustExec("drop table if exists t2")
+	tbl, err = s.getClonedTable("test", "t1")
+	c.Assert(err, IsNil)
+	c.Assert(tbl.PlacementPolicyRef, NotNil)
+	tbl.Name = model.NewCIStr("t2")
+	tbl.PlacementPolicyRef.Name = model.NewCIStr("pxx")
+	err = s.dom.DDL().CreateTableWithInfo(tk.Se, model.NewCIStr("test"), tbl, ddl.OnExistError)
+	c.Assert(err, IsNil)
+	tk.MustQuery("show create table t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
+		"  `id` int(11) DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='aaa'"))
+
+	// createSchemaWithInfo in ignore mode
+	tk.MustExec("drop database if exists db2")
+	db1, ok := s.getClonedDatabase("db1")
+	c.Assert(ok, IsTrue)
+	c.Assert(db1.PlacementPolicyRef, NotNil)
+	db1.Name = model.NewCIStr("db2")
+	err = s.dom.DDL().CreateSchemaWithInfo(tk.Se, db1, ddl.OnExistError)
+	c.Assert(err, IsNil)
+	tk.MustQuery("show create database db2").Check(testkit.Rows("db2 CREATE DATABASE `db2` /*!40100 DEFAULT CHARACTER SET utf8mb4 */"))
+
+	// createSchemaWithInfo in ignore mode (policy not exists)
+	tk.MustExec("drop database if exists db2")
+	db1, ok = s.getClonedDatabase("db1")
+	c.Assert(ok, IsTrue)
+	c.Assert(db1.PlacementPolicyRef, NotNil)
+	db1.Name = model.NewCIStr("db2")
+	db1.PlacementPolicyRef.Name = model.NewCIStr("pxx")
+	err = s.dom.DDL().CreateSchemaWithInfo(tk.Se, db1, ddl.OnExistError)
+	c.Assert(err, IsNil)
+	tk.MustQuery("show create database db2").Check(testkit.Rows("db2 CREATE DATABASE `db2` /*!40100 DEFAULT CHARACTER SET utf8mb4 */"))
+
 }
 
 func (s *testDBSuite6) TestPlacementTiflashCheck(c *C) {

expression/distsql_builtin.go

@@ -946,6 +946,8 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti
 		f = &builtinCharSig{base}
 	case tipb.ScalarFuncSig_CharLengthUTF8:
 		f = &builtinCharLengthUTF8Sig{base}
+	case tipb.ScalarFuncSig_CharLength:
+		f = &builtinCharLengthBinarySig{base}
 	case tipb.ScalarFuncSig_Concat:
 		f = &builtinConcatSig{base, maxAllowedPacket}
 	case tipb.ScalarFuncSig_ConcatWS:

expression/integration_test.go

@@ -810,6 +810,12 @@ func TestStringBuiltin(t *testing.T) {
 	result = tk.MustQuery("select a,b,concat_ws(',',a,b) from t")
 	result.Check(testkit.Rows("114.57011441 38.04620115 114.57011441,38.04620115",
 		"-38.04620119 38.04620115 -38.04620119,38.04620115"))
+
+	// For issue 31603, only affects unistore.
+	tk.MustExec("drop table if exists t1;")
+	tk.MustExec("create table t1(c1 varbinary(100));")
+	tk.MustExec("insert into t1 values('abc');")
+	tk.MustQuery("select 1 from t1 where char_length(c1) = 10;").Check(testkit.Rows())
 }
 
 func TestInvalidStrings(t *testing.T) {

parser/ast/ast.go

@@ -18,6 +18,7 @@ package ast
 import (
 	"io"
 
+	"github.com/pingcap/tidb/parser/charset"
 	"github.com/pingcap/tidb/parser/format"
 	"github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tidb/parser/types"
@@ -37,10 +38,12 @@ type Node interface {
 	// children should be skipped. Otherwise, call its children in particular order that
 	// later elements depends on former elements. Finally, return visitor.Leave.
 	Accept(v Visitor) (node Node, ok bool)
-	// Text returns the original text of the element.
+	// Text returns the utf8 encoding text of the element.
 	Text() string
+	// OriginalText returns the original text of the element.
+	OriginalText() string
 	// SetText sets original text to the Node.
-	SetText(text string)
+	SetText(enc charset.Encoding, text string)
 	// SetOriginTextPosition set the start offset of this node in the origin text.
 	SetOriginTextPosition(offset int)
 	// OriginTextPosition get the start offset of this node in the origin text.

parser/ast/base.go

@@ -14,12 +14,19 @@
 package ast
 
 import (
+	"sync"
+
+	"github.com/pingcap/tidb/parser/charset"
 	"github.com/pingcap/tidb/parser/types"
 )
 
 // node is the struct implements Node interface except for Accept method.
 // Node implementations should embed it in.
 type node struct {
+	utf8Text string
+	enc      charset.Encoding
+	once     *sync.Once
+
 	text   string
 	offset int
 }
@@ -35,12 +42,30 @@ func (n *node) OriginTextPosition() int {
 }
 
 // SetText implements Node interface.
-func (n *node) SetText(text string) {
+func (n *node) SetText(enc charset.Encoding, text string) {
+	n.enc = enc
 	n.text = text
+	n.once = &sync.Once{}
 }
 
 // Text implements Node interface.
 func (n *node) Text() string {
+	if n.once == nil {
+		return n.text
+	}
+	n.once.Do(func() {
+		if n.enc == nil {
+			n.utf8Text = n.text
+			return
+		}
+		utf8Lit, _ := n.enc.Transform(nil, charset.HackSlice(n.text), charset.OpDecodeReplace)
+		n.utf8Text = charset.HackString(utf8Lit)
+	})
+	return n.utf8Text
+}
+
+// OriginalText implements Node interface.
+func (n *node) OriginalText() string {
 	return n.text
 }
 

parser/ast/base_test.go

@@ -0,0 +1,42 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package ast is the abstract syntax tree parsed from a SQL statement by parser.
+// It can be analysed and transformed by optimizer.
+package ast
+
+import (
+	"testing"
+
+	"github.com/pingcap/tidb/parser/charset"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNodeSetText(t *testing.T) {
+	n := &node{}
+	tests := []struct {
+		text           string
+		enc            charset.Encoding
+		expectUTF8Text string
+		expectText     string
+	}{
+		{"你好", nil, "你好", "你好"},
+		{"\xd2\xbb", charset.EncodingGBKImpl, "一", "\xd2\xbb"},
+		{"\xc1\xd0", charset.EncodingGBKImpl, "列", "\xc1\xd0"},
+	}
+	for _, tt := range tests {
+		n.SetText(tt.enc, tt.text)
+		require.Equal(t, tt.expectUTF8Text, n.Text())
+		require.Equal(t, tt.expectText, n.OriginalText())
+	}
+}

parser/ast/util_test.go

@@ -120,7 +120,7 @@ type nodeTextCleaner struct {
 
 // Enter implements Visitor interface.
 func (checker *nodeTextCleaner) Enter(in Node) (out Node, skipChildren bool) {
-	in.SetText("")
+	in.SetText(nil, "")
 	in.SetOriginTextPosition(0)
 	switch node := in.(type) {
 	case *Constraint: