diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index 2dbf7e5e4c6..a0873fa0670 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -125,13 +125,17 @@ func fetchCacheEnvironment(version collver) *Environment { // The version string must be in the format that is sent by the server as the version packet // when opening a new MySQL connection func NewEnvironment(serverVersion string) *Environment { - var version collver = collverMySQL56 + // 5.7 is the oldest version we support today, so use that as + // the default. + // NOTE: this should be changed when we EOL MySQL 5.7 support + var version collver = collverMySQL57 + serverVersion = strings.TrimSpace(strings.ToLower(serverVersion)) switch { case strings.HasSuffix(serverVersion, "-ripple"): // the ripple binlog server can mask the actual version of mysqld; // assume we have the highest version = collverMySQL80 - case strings.Contains(serverVersion, "MariaDB"): + case strings.Contains(serverVersion, "mariadb"): switch { case strings.Contains(serverVersion, "10.0."): version = collverMariaDB100 diff --git a/go/mysql/collations/integration/collations_test.go b/go/mysql/collations/integration/collations_test.go index 89dd4ace2a4..e91719090bc 100644 --- a/go/mysql/collations/integration/collations_test.go +++ b/go/mysql/collations/integration/collations_test.go @@ -35,9 +35,19 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/remote" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/sqlparser" ) +var collationEnv *collations.Environment + +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func getSQLQueries(t *testing.T, testfile string) []string { tf, err := os.Open(testfile) if err != nil { @@ -105,7 +115,7 @@ func parseWeightString(b []byte) []byte { } func (u *uca900CollationTest) Test(t *testing.T, result *sqltypes.Result) { - coll := collations.Local().LookupByName(u.collation) + coll := collationEnv.LookupByName(u.collation) if coll == nil { t.Fatalf("unknown collation %q", u.collation) } @@ -215,7 +225,7 @@ func TestCollationWithSpace(t *testing.T) { for _, collName := range []string{"utf8mb4_0900_ai_ci", "utf8mb4_unicode_ci", "utf8mb4_unicode_520_ci"} { t.Run(collName, func(t *testing.T) { - local := collations.Local().LookupByName(collName) + local := collationEnv.LookupByName(collName) remote := remote.NewCollation(conn, collName) for _, size := range []int{0, codepoints, codepoints + 1, codepoints + 2, 20, 32} { diff --git a/go/mysql/collations/local.go b/go/mysql/collations/local.go index a656a3dad2e..7c3b2ab0b15 100644 --- a/go/mysql/collations/local.go +++ b/go/mysql/collations/local.go @@ -36,7 +36,10 @@ func Local() *Environment { panic("collations.Local() called too early") } if *servenv.MySQLServerVersion == "" { - defaultEnv = fetchCacheEnvironment(collverMySQL80) + // The default server version used by vtgate is 5.7.9 + // NOTE: this should be changed along with the effective default + // for the vtgate mysql_server_version flag. + defaultEnv = fetchCacheEnvironment(collverMySQL57) } else { defaultEnv = NewEnvironment(*servenv.MySQLServerVersion) } diff --git a/go/mysql/endtoend/query_test.go b/go/mysql/endtoend/query_test.go index 1118c46f85a..f6410147a74 100644 --- a/go/mysql/endtoend/query_test.go +++ b/go/mysql/endtoend/query_test.go @@ -33,6 +33,10 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" ) +const ( + charsetName = "utf8mb4" +) + func columnSize(cs collations.ID, size uint32) uint32 { // utf8_general_ci results in smaller max column sizes because MySQL 5.7 is silly if collations.Local().LookupByID(cs).Charset().Name() == "utf8mb3" { @@ -78,6 +82,7 @@ func TestQueries(t *testing.T) { if err != nil { t.Fatalf("insert failed: %v", err) } + collID := getDefaultCollationID() expectedResult := &sqltypes.Result{ Fields: []*querypb.Field{ { @@ -101,8 +106,8 @@ func TestQueries(t *testing.T) { OrgTable: "a", Database: "vttest", OrgName: "name", - ColumnLength: columnSize(conn.CharacterSet, 512), - Charset: uint32(conn.CharacterSet), + ColumnLength: columnSize(collID, 512), + Charset: uint32(collID), }, }, Rows: [][]sqltypes.Value{ @@ -188,6 +193,7 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) { } // Check the fields. + collID := getDefaultCollationID() expectedFields := []*querypb.Field{ { Name: "id", @@ -210,8 +216,8 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) { OrgTable: "a", Database: "vttest", OrgName: "name", - ColumnLength: columnSize(conn.CharacterSet, 512), - Charset: uint32(conn.CharacterSet), + ColumnLength: columnSize(collID, 512), + Charset: uint32(collID), }, } fields, err := conn.Fields() @@ -306,7 +312,8 @@ func TestSysInfo(t *testing.T) { _, err = conn.ExecuteFetch("drop table if exists `a`", 1000, true) require.NoError(t, err) - _, err = conn.ExecuteFetch("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY (`one`,`two`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", 1000, true) + _, err = conn.ExecuteFetch(fmt.Sprintf("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY (`one`,`two`)) ENGINE=InnoDB DEFAULT CHARSET=%s", + charsetName), 1000, true) require.NoError(t, err) defer conn.ExecuteFetch("drop table `a`", 1000, true) @@ -339,3 +346,9 @@ func TestSysInfo(t *testing.T) { assert.EqualValues(t, sqltypes.Uint64, qr.Fields[4].Type) assert.EqualValues(t, querypb.Type_UINT64, qr.Rows[0][4].Type()) } + +func getDefaultCollationID() collations.ID { + collationHandler := collations.Local() + collation := collationHandler.DefaultCollationForCharset(charsetName) + return collation.ID() +} diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index d460c387bbc..bfba50e3c1a 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -32,6 +32,7 @@ import ( "time" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/vtgate/planbuilder" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" ) @@ -69,7 +70,6 @@ const defaultVtGatePlannerVersion = planbuilder.Gen4CompareV3 // Setup starts Vtgate process with required arguements func (vtgate *VtgateProcess) Setup() (err error) { - args := []string{ "--topo_implementation", vtgate.CommonArg.TopoImplementation, "--topo_global_server_address", vtgate.CommonArg.TopoGlobalAddress, @@ -86,6 +86,28 @@ func (vtgate *VtgateProcess) Setup() (err error) { "--service_map", vtgate.ServiceMap, "--mysql_auth_server_impl", vtgate.MySQLAuthServerImpl, } + // If no explicit mysql_server_version has been specified then we autodetect + // the MySQL version that will be used for the test and base the vtgate's + // mysql server version on that. + msvflag := false + for _, f := range vtgate.ExtraArgs { + if strings.Contains(f, "mysql_server_version") { + msvflag = true + break + } + } + if !msvflag { + version, err := mysqlctl.GetVersionString() + if err != nil { + return err + } + _, vers, err := mysqlctl.ParseVersionString(version) + if err != nil { + return err + } + mysqlvers := fmt.Sprintf("%d.%d.%d-vitess", vers.Major, vers.Minor, vers.Patch) + args = append(args, "--mysql_server_version", mysqlvers) + } if vtgate.PlannerVersion > 0 { args = append(args, "--planner-version", vtgate.PlannerVersion.String()) } diff --git a/go/test/endtoend/vtgate/reservedconn/main_test.go b/go/test/endtoend/vtgate/reservedconn/main_test.go index 08f7f544d51..2dcd66109fc 100644 --- a/go/test/endtoend/vtgate/reservedconn/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/main_test.go @@ -124,6 +124,8 @@ func TestMain(m *testing.M) { } // Start vtgate + // This test requires setting the mysql_server_version vtgate flag + // to 5.7 regardless of the actual MySQL version used for the tests. clusterInstance.VtGateExtraArgs = []string{"--lock_heartbeat_time", "2s", "--mysql_server_version", "5.7.0"} if err := clusterInstance.StartVtgate(); err != nil { return 1 diff --git a/go/vt/vtgate/engine/memory_sort_test.go b/go/vt/vtgate/engine/memory_sort_test.go index f24b968e378..80b75b61774 100644 --- a/go/vt/vtgate/engine/memory_sort_test.go +++ b/go/vt/vtgate/engine/memory_sort_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/mysql/collations" @@ -31,6 +32,13 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" ) +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func TestMemorySortExecute(t *testing.T) { fields := sqltypes.MakeTestFields( "c1|c2", diff --git a/go/vt/vtgate/engine/ordered_aggregate_test.go b/go/vt/vtgate/engine/ordered_aggregate_test.go index 6e288cf6d56..461df177963 100644 --- a/go/vt/vtgate/engine/ordered_aggregate_test.go +++ b/go/vt/vtgate/engine/ordered_aggregate_test.go @@ -31,8 +31,18 @@ import ( binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/servenv" ) +var collationEnv *collations.Environment + +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func TestOrderedAggregateExecute(t *testing.T) { assert := assert.New(t) fields := sqltypes.MakeTestFields( @@ -999,7 +1009,7 @@ func TestOrderedAggregateCollate(t *testing.T) { )}, } - collationID, _ := collations.Local().LookupID("utf8mb4_0900_ai_ci") + collationID, _ := collationEnv.LookupID("utf8mb4_0900_ai_ci") oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ Opcode: AggregateSum, @@ -1042,7 +1052,7 @@ func TestOrderedAggregateCollateAS(t *testing.T) { )}, } - collationID, _ := collations.Local().LookupID("utf8mb4_0900_as_ci") + collationID, _ := collationEnv.LookupID("utf8mb4_0900_as_ci") oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ Opcode: AggregateSum, @@ -1087,7 +1097,7 @@ func TestOrderedAggregateCollateKS(t *testing.T) { )}, } - collationID, _ := collations.Local().LookupID("utf8mb4_ja_0900_as_cs_ks") + collationID, _ := collationEnv.LookupID("utf8mb4_ja_0900_as_cs_ks") oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ Opcode: AggregateSum, diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index 569bb8d4c05..e7d58d66cd3 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/mysql/collations" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/sqlparser" @@ -49,6 +50,13 @@ var defaultSelectResult = sqltypes.MakeTestResult( "1", ) +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func TestSelectUnsharded(t *testing.T) { sel := NewRoute( Unsharded, diff --git a/go/vt/vtgate/evalengine/comparisons_test.go b/go/vt/vtgate/evalengine/comparisons_test.go index 27be0cff2cb..68daa64024f 100644 --- a/go/vt/vtgate/evalengine/comparisons_test.go +++ b/go/vt/vtgate/evalengine/comparisons_test.go @@ -26,6 +26,7 @@ import ( "vitess.io/vitess/go/mysql/collations" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -45,13 +46,21 @@ type testCase struct { } var ( - T = true - F = false + T = true + F = false + collationEnv *collations.Environment ) +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func defaultCollation() collations.TypedCollation { return collations.TypedCollation{ - Collation: collations.Local().LookupByName("utf8mb4_bin").ID(), + Collation: collationEnv.LookupByName("utf8mb4_bin").ID(), Coercibility: collations.CoerceImplicit, Repertoire: collations.RepertoireASCII, } @@ -1053,7 +1062,7 @@ func TestNullComparisons(t *testing.T) { } func TestNullsafeCompare(t *testing.T) { - collation := collations.Local().LookupByName("utf8mb4_general_ci").ID() + collation := collationEnv.LookupByName("utf8mb4_general_ci").ID() tcases := []struct { v1, v2 sqltypes.Value out int @@ -1155,7 +1164,7 @@ func TestNullsafeCompare(t *testing.T) { } func getCollationID(collation string) collations.ID { - id, _ := collations.Local().LookupID(collation) + id, _ := collationEnv.LookupID(collation) return id } diff --git a/go/vt/vtgate/evalengine/integration/comparison_test.go b/go/vt/vtgate/evalengine/integration/comparison_test.go index f55d5881aec..cecf7dc4a42 100644 --- a/go/vt/vtgate/evalengine/integration/comparison_test.go +++ b/go/vt/vtgate/evalengine/integration/comparison_test.go @@ -28,9 +28,19 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vtgate/evalengine" ) +var collationEnv *collations.Environment + +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func perm(a []string, f func([]string)) { perm1(a, f, 0) } @@ -126,7 +136,7 @@ func compareRemoteExpr(t *testing.T, conn *mysql.Conn, expr string) { // TODO: passthrough proper collations for nullable fields remoteCollation = collations.CollationBinaryID } else { - remoteCollation = collations.Local().LookupByName(remote.Rows[0][1].ToString()).ID() + remoteCollation = collationEnv.LookupByName(remote.Rows[0][1].ToString()).ID() } } } diff --git a/go/vt/vttablet/tabletmanager/tm_init_test.go b/go/vt/vttablet/tabletmanager/tm_init_test.go index 90a138977c6..cccdcb91d70 100644 --- a/go/vt/vttablet/tabletmanager/tm_init_test.go +++ b/go/vt/vttablet/tabletmanager/tm_init_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/sync2" @@ -42,6 +43,12 @@ import ( vschemapb "vitess.io/vitess/go/vt/proto/vschema" ) +var ( + dbServerVersion = "5.7.0" + charsetName = "utf8mb4" + dbsvCollID = collations.NewEnvironment(dbServerVersion).DefaultCollationForCharset(charsetName).ID() +) + func TestStartBuildTabletFromInput(t *testing.T) { alias := &topodatapb.TabletAlias{ Cell: "cell", @@ -49,7 +56,6 @@ func TestStartBuildTabletFromInput(t *testing.T) { } port := int32(12) grpcport := int32(34) - dbServerVersion := "5.7.0" // Hostname should be used as is. *tabletHostname = "foo" @@ -71,7 +77,7 @@ func TestStartBuildTabletFromInput(t *testing.T) { Tags: map[string]string{}, DbNameOverride: "aa", DbServerVersion: dbServerVersion, - DefaultConnCollation: 255, + DefaultConnCollation: uint32(dbsvCollID), } gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil) @@ -131,7 +137,6 @@ func TestBuildTabletFromInputWithBuildTags(t *testing.T) { } port := int32(12) grpcport := int32(34) - dbServerVersion := "5.7.0" // Hostname should be used as is. *tabletHostname = "foo" @@ -154,8 +159,8 @@ func TestBuildTabletFromInputWithBuildTags(t *testing.T) { Type: topodatapb.TabletType_REPLICA, Tags: servenv.AppVersion.ToStringMap(), DbNameOverride: "aa", - DbServerVersion: "5.7.0", - DefaultConnCollation: 255, + DbServerVersion: dbServerVersion, + DefaultConnCollation: uint32(dbsvCollID), } gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)