From fe1e0b87b20431a1d2a1f117f5a221b336f5f011 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 15 Jul 2022 02:00:39 -0400 Subject: [PATCH 01/12] Auto detect MySQL version and use in vtgate --mysql_server_version flag Signed-off-by: Matt Lord --- go/test/endtoend/cluster/vtgate_process.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index d460c387bbc..7d7c7e15b4c 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -32,6 +32,7 @@ import ( "time" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/vtgate/planbuilder" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" ) @@ -69,6 +70,15 @@ const defaultVtGatePlannerVersion = planbuilder.Gen4CompareV3 // Setup starts Vtgate process with required arguements func (vtgate *VtgateProcess) Setup() (err error) { + version, err := mysqlctl.GetVersionString() + if err != nil { + return err + } + _, vers, err := mysqlctl.ParseVersionString(version) + if err != nil { + return err + } + mysqlvers := fmt.Sprintf("%d.%d.%d-vitess", vers.Major, vers.Minor, vers.Patch) args := []string{ "--topo_implementation", vtgate.CommonArg.TopoImplementation, @@ -85,6 +95,7 @@ func (vtgate *VtgateProcess) Setup() (err error) { "--tablet_types_to_wait", vtgate.TabletTypesToWait, "--service_map", vtgate.ServiceMap, "--mysql_auth_server_impl", vtgate.MySQLAuthServerImpl, + "--mysql_server_version", mysqlvers, } if vtgate.PlannerVersion > 0 { args = append(args, "--planner-version", vtgate.PlannerVersion.String()) From c5cf717478d2575a1e3d7f183aad56e95b85009b Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 15 Jul 2022 02:26:54 -0400 Subject: [PATCH 02/12] Remove any explicit static usage of mysql_server_version Signed-off-by: Matt Lord --- go/test/endtoend/mysqlserver/main_test.go | 1 - go/test/endtoend/preparestmt/main_test.go | 1 - go/test/endtoend/vtgate/mysql80/main_test.go | 1 - go/test/endtoend/vtgate/reservedconn/main_test.go | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go/test/endtoend/mysqlserver/main_test.go b/go/test/endtoend/mysqlserver/main_test.go index 42b4e6ea235..96fc9e2f910 100644 --- a/go/test/endtoend/mysqlserver/main_test.go +++ b/go/test/endtoend/mysqlserver/main_test.go @@ -113,7 +113,6 @@ func TestMain(m *testing.M) { "--mysql_server_query_timeout", "1s", "--mysql_auth_server_impl", "static", "--mysql_auth_server_static_file", clusterInstance.TmpDirectory + mysqlAuthServerStatic, - "--mysql_server_version", "8.0.16-7", "--warn_sharded_only=true", } diff --git a/go/test/endtoend/preparestmt/main_test.go b/go/test/endtoend/preparestmt/main_test.go index 018e9d266fd..ffd7ab79aa4 100644 --- a/go/test/endtoend/preparestmt/main_test.go +++ b/go/test/endtoend/preparestmt/main_test.go @@ -204,7 +204,6 @@ func TestMain(m *testing.M) { vtgateInstance.ExtraArgs = []string{ "--mysql_server_query_timeout", "1s", "--mysql_auth_server_static_file", clusterInstance.TmpDirectory + "/" + mysqlAuthServerStatic, - "--mysql_server_version", "8.0.16-7", } // Start vtgate diff --git a/go/test/endtoend/vtgate/mysql80/main_test.go b/go/test/endtoend/vtgate/mysql80/main_test.go index 4f5897d1f59..603ea9b94dc 100644 --- a/go/test/endtoend/vtgate/mysql80/main_test.go +++ b/go/test/endtoend/vtgate/mysql80/main_test.go @@ -60,7 +60,6 @@ func TestMain(m *testing.M) { clusterInstance.VtGatePlannerVersion = querypb.ExecuteOptions_Gen4 clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, "--enable_system_settings=true", - "--mysql_server_version=8.0.16-7", ) // Start vtgate err = clusterInstance.StartVtgate() diff --git a/go/test/endtoend/vtgate/reservedconn/main_test.go b/go/test/endtoend/vtgate/reservedconn/main_test.go index 08f7f544d51..0322161849a 100644 --- a/go/test/endtoend/vtgate/reservedconn/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/main_test.go @@ -124,7 +124,7 @@ func TestMain(m *testing.M) { } // Start vtgate - clusterInstance.VtGateExtraArgs = []string{"--lock_heartbeat_time", "2s", "--mysql_server_version", "5.7.0"} + clusterInstance.VtGateExtraArgs = []string{"--lock_heartbeat_time", "2s"} if err := clusterInstance.StartVtgate(); err != nil { return 1 } From cbee18a1038d11a2ec74b49bd9b96628fecd1dba Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 15 Jul 2022 03:06:19 -0400 Subject: [PATCH 03/12] The reservedconns test needs to override the mysql_server_version Signed-off-by: Matt Lord --- go/test/endtoend/vtgate/reservedconn/main_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vtgate/reservedconn/main_test.go b/go/test/endtoend/vtgate/reservedconn/main_test.go index 0322161849a..2dcd66109fc 100644 --- a/go/test/endtoend/vtgate/reservedconn/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/main_test.go @@ -124,7 +124,9 @@ func TestMain(m *testing.M) { } // Start vtgate - clusterInstance.VtGateExtraArgs = []string{"--lock_heartbeat_time", "2s"} + // This test requires setting the mysql_server_version vtgate flag + // to 5.7 regardless of the actual MySQL version used for the tests. + clusterInstance.VtGateExtraArgs = []string{"--lock_heartbeat_time", "2s", "--mysql_server_version", "5.7.0"} if err := clusterInstance.StartVtgate(); err != nil { return 1 } From b1820df2134a47002568b46a78ba16394edbe016 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 01:06:03 -0400 Subject: [PATCH 04/12] Explicitly allow overriding the mysql_system_version Signed-off-by: Matt Lord --- go/test/endtoend/cluster/vtgate_process.go | 33 +++++++++++++------- go/test/endtoend/mysqlserver/main_test.go | 1 + go/test/endtoend/preparestmt/main_test.go | 1 + go/test/endtoend/vtgate/mysql80/main_test.go | 1 + 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index 7d7c7e15b4c..bfba50e3c1a 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -70,16 +70,6 @@ const defaultVtGatePlannerVersion = planbuilder.Gen4CompareV3 // Setup starts Vtgate process with required arguements func (vtgate *VtgateProcess) Setup() (err error) { - version, err := mysqlctl.GetVersionString() - if err != nil { - return err - } - _, vers, err := mysqlctl.ParseVersionString(version) - if err != nil { - return err - } - mysqlvers := fmt.Sprintf("%d.%d.%d-vitess", vers.Major, vers.Minor, vers.Patch) - args := []string{ "--topo_implementation", vtgate.CommonArg.TopoImplementation, "--topo_global_server_address", vtgate.CommonArg.TopoGlobalAddress, @@ -95,7 +85,28 @@ func (vtgate *VtgateProcess) Setup() (err error) { "--tablet_types_to_wait", vtgate.TabletTypesToWait, "--service_map", vtgate.ServiceMap, "--mysql_auth_server_impl", vtgate.MySQLAuthServerImpl, - "--mysql_server_version", mysqlvers, + } + // If no explicit mysql_server_version has been specified then we autodetect + // the MySQL version that will be used for the test and base the vtgate's + // mysql server version on that. + msvflag := false + for _, f := range vtgate.ExtraArgs { + if strings.Contains(f, "mysql_server_version") { + msvflag = true + break + } + } + if !msvflag { + version, err := mysqlctl.GetVersionString() + if err != nil { + return err + } + _, vers, err := mysqlctl.ParseVersionString(version) + if err != nil { + return err + } + mysqlvers := fmt.Sprintf("%d.%d.%d-vitess", vers.Major, vers.Minor, vers.Patch) + args = append(args, "--mysql_server_version", mysqlvers) } if vtgate.PlannerVersion > 0 { args = append(args, "--planner-version", vtgate.PlannerVersion.String()) diff --git a/go/test/endtoend/mysqlserver/main_test.go b/go/test/endtoend/mysqlserver/main_test.go index 96fc9e2f910..dd3a97be9f9 100644 --- a/go/test/endtoend/mysqlserver/main_test.go +++ b/go/test/endtoend/mysqlserver/main_test.go @@ -114,6 +114,7 @@ func TestMain(m *testing.M) { "--mysql_auth_server_impl", "static", "--mysql_auth_server_static_file", clusterInstance.TmpDirectory + mysqlAuthServerStatic, "--warn_sharded_only=true", + "--mysql_server_version", "8.0.16-7", } clusterInstance.VtTabletExtraArgs = []string{ diff --git a/go/test/endtoend/preparestmt/main_test.go b/go/test/endtoend/preparestmt/main_test.go index ffd7ab79aa4..018e9d266fd 100644 --- a/go/test/endtoend/preparestmt/main_test.go +++ b/go/test/endtoend/preparestmt/main_test.go @@ -204,6 +204,7 @@ func TestMain(m *testing.M) { vtgateInstance.ExtraArgs = []string{ "--mysql_server_query_timeout", "1s", "--mysql_auth_server_static_file", clusterInstance.TmpDirectory + "/" + mysqlAuthServerStatic, + "--mysql_server_version", "8.0.16-7", } // Start vtgate diff --git a/go/test/endtoend/vtgate/mysql80/main_test.go b/go/test/endtoend/vtgate/mysql80/main_test.go index 603ea9b94dc..4f5897d1f59 100644 --- a/go/test/endtoend/vtgate/mysql80/main_test.go +++ b/go/test/endtoend/vtgate/mysql80/main_test.go @@ -60,6 +60,7 @@ func TestMain(m *testing.M) { clusterInstance.VtGatePlannerVersion = querypb.ExecuteOptions_Gen4 clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, "--enable_system_settings=true", + "--mysql_server_version=8.0.16-7", ) // Start vtgate err = clusterInstance.StartVtgate() From f143445537b3a9b17aa98fac5ddefc0fddd8a7af Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 01:07:27 -0400 Subject: [PATCH 05/12] minimize diff Signed-off-by: Matt Lord --- go/test/endtoend/mysqlserver/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/mysqlserver/main_test.go b/go/test/endtoend/mysqlserver/main_test.go index dd3a97be9f9..42b4e6ea235 100644 --- a/go/test/endtoend/mysqlserver/main_test.go +++ b/go/test/endtoend/mysqlserver/main_test.go @@ -113,8 +113,8 @@ func TestMain(m *testing.M) { "--mysql_server_query_timeout", "1s", "--mysql_auth_server_impl", "static", "--mysql_auth_server_static_file", clusterInstance.TmpDirectory + mysqlAuthServerStatic, - "--warn_sharded_only=true", "--mysql_server_version", "8.0.16-7", + "--warn_sharded_only=true", } clusterInstance.VtTabletExtraArgs = []string{ From bbadd17463c57914a60225aa0ab2d7c06b3c5ccf Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 14:39:41 -0400 Subject: [PATCH 06/12] Fix https://github.com/vitessio/vitess/issues/10712 Signed-off-by: Matt Lord --- go/mysql/collations/env.go | 5 ++++- go/mysql/collations/local.go | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index 2dbf7e5e4c6..21d286073fa 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -125,7 +125,10 @@ func fetchCacheEnvironment(version collver) *Environment { // The version string must be in the format that is sent by the server as the version packet // when opening a new MySQL connection func NewEnvironment(serverVersion string) *Environment { - var version collver = collverMySQL56 + // 5.7 is the oldest version we support today + // NOTE: this should be changed when we EOL MySQL 5.7 support + var version collver = collverMySQL57 + serverVersion = strings.TrimSpace(serverVersion) switch { case strings.HasSuffix(serverVersion, "-ripple"): // the ripple binlog server can mask the actual version of mysqld; diff --git a/go/mysql/collations/local.go b/go/mysql/collations/local.go index a656a3dad2e..34f13b6db60 100644 --- a/go/mysql/collations/local.go +++ b/go/mysql/collations/local.go @@ -20,6 +20,7 @@ package collations import ( "flag" + "strings" "sync" "vitess.io/vitess/go/vt/servenv" @@ -36,9 +37,12 @@ func Local() *Environment { panic("collations.Local() called too early") } if *servenv.MySQLServerVersion == "" { - defaultEnv = fetchCacheEnvironment(collverMySQL80) + // The default server version used by vtgate is 5.7.9 + // NOTE: this should be changed along with the effective default + // for the vtgate mysql_server_version flag. + defaultEnv = fetchCacheEnvironment(collverMySQL57) } else { - defaultEnv = NewEnvironment(*servenv.MySQLServerVersion) + defaultEnv = NewEnvironment(strings.Clone(*servenv.MySQLServerVersion)) } }) return defaultEnv From 2f1a0788a6b0a25f5dfb5e14159b1c962e10444e Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 14:54:16 -0400 Subject: [PATCH 07/12] Use more robust string comparisons Signed-off-by: Matt Lord --- go/mysql/collations/env.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index 21d286073fa..a0873fa0670 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -125,16 +125,17 @@ func fetchCacheEnvironment(version collver) *Environment { // The version string must be in the format that is sent by the server as the version packet // when opening a new MySQL connection func NewEnvironment(serverVersion string) *Environment { - // 5.7 is the oldest version we support today + // 5.7 is the oldest version we support today, so use that as + // the default. // NOTE: this should be changed when we EOL MySQL 5.7 support var version collver = collverMySQL57 - serverVersion = strings.TrimSpace(serverVersion) + serverVersion = strings.TrimSpace(strings.ToLower(serverVersion)) switch { case strings.HasSuffix(serverVersion, "-ripple"): // the ripple binlog server can mask the actual version of mysqld; // assume we have the highest version = collverMySQL80 - case strings.Contains(serverVersion, "MariaDB"): + case strings.Contains(serverVersion, "mariadb"): switch { case strings.Contains(serverVersion, "10.0."): version = collverMariaDB100 From 7a8afcf895bdd5d795aef5d4545ef6cb899719b6 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 19:20:41 -0400 Subject: [PATCH 08/12] Fix collation handling in tests Signed-off-by: Matt Lord --- go/mysql/endtoend/query_test.go | 23 ++++++++++++++++----- go/vt/vtgate/evalengine/comparisons_test.go | 19 ++++++++++++----- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/go/mysql/endtoend/query_test.go b/go/mysql/endtoend/query_test.go index 1118c46f85a..f6410147a74 100644 --- a/go/mysql/endtoend/query_test.go +++ b/go/mysql/endtoend/query_test.go @@ -33,6 +33,10 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" ) +const ( + charsetName = "utf8mb4" +) + func columnSize(cs collations.ID, size uint32) uint32 { // utf8_general_ci results in smaller max column sizes because MySQL 5.7 is silly if collations.Local().LookupByID(cs).Charset().Name() == "utf8mb3" { @@ -78,6 +82,7 @@ func TestQueries(t *testing.T) { if err != nil { t.Fatalf("insert failed: %v", err) } + collID := getDefaultCollationID() expectedResult := &sqltypes.Result{ Fields: []*querypb.Field{ { @@ -101,8 +106,8 @@ func TestQueries(t *testing.T) { OrgTable: "a", Database: "vttest", OrgName: "name", - ColumnLength: columnSize(conn.CharacterSet, 512), - Charset: uint32(conn.CharacterSet), + ColumnLength: columnSize(collID, 512), + Charset: uint32(collID), }, }, Rows: [][]sqltypes.Value{ @@ -188,6 +193,7 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) { } // Check the fields. + collID := getDefaultCollationID() expectedFields := []*querypb.Field{ { Name: "id", @@ -210,8 +216,8 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) { OrgTable: "a", Database: "vttest", OrgName: "name", - ColumnLength: columnSize(conn.CharacterSet, 512), - Charset: uint32(conn.CharacterSet), + ColumnLength: columnSize(collID, 512), + Charset: uint32(collID), }, } fields, err := conn.Fields() @@ -306,7 +312,8 @@ func TestSysInfo(t *testing.T) { _, err = conn.ExecuteFetch("drop table if exists `a`", 1000, true) require.NoError(t, err) - _, err = conn.ExecuteFetch("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY (`one`,`two`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", 1000, true) + _, err = conn.ExecuteFetch(fmt.Sprintf("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY (`one`,`two`)) ENGINE=InnoDB DEFAULT CHARSET=%s", + charsetName), 1000, true) require.NoError(t, err) defer conn.ExecuteFetch("drop table `a`", 1000, true) @@ -339,3 +346,9 @@ func TestSysInfo(t *testing.T) { assert.EqualValues(t, sqltypes.Uint64, qr.Fields[4].Type) assert.EqualValues(t, querypb.Type_UINT64, qr.Rows[0][4].Type()) } + +func getDefaultCollationID() collations.ID { + collationHandler := collations.Local() + collation := collationHandler.DefaultCollationForCharset(charsetName) + return collation.ID() +} diff --git a/go/vt/vtgate/evalengine/comparisons_test.go b/go/vt/vtgate/evalengine/comparisons_test.go index 27be0cff2cb..68daa64024f 100644 --- a/go/vt/vtgate/evalengine/comparisons_test.go +++ b/go/vt/vtgate/evalengine/comparisons_test.go @@ -26,6 +26,7 @@ import ( "vitess.io/vitess/go/mysql/collations" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -45,13 +46,21 @@ type testCase struct { } var ( - T = true - F = false + T = true + F = false + collationEnv *collations.Environment ) +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func defaultCollation() collations.TypedCollation { return collations.TypedCollation{ - Collation: collations.Local().LookupByName("utf8mb4_bin").ID(), + Collation: collationEnv.LookupByName("utf8mb4_bin").ID(), Coercibility: collations.CoerceImplicit, Repertoire: collations.RepertoireASCII, } @@ -1053,7 +1062,7 @@ func TestNullComparisons(t *testing.T) { } func TestNullsafeCompare(t *testing.T) { - collation := collations.Local().LookupByName("utf8mb4_general_ci").ID() + collation := collationEnv.LookupByName("utf8mb4_general_ci").ID() tcases := []struct { v1, v2 sqltypes.Value out int @@ -1155,7 +1164,7 @@ func TestNullsafeCompare(t *testing.T) { } func getCollationID(collation string) collations.ID { - id, _ := collations.Local().LookupID(collation) + id, _ := collationEnv.LookupID(collation) return id } From 133b471bafc87582f8bba7f18d8c93ad4f36bca8 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 19:54:47 -0400 Subject: [PATCH 09/12] Fix tm_init_test Signed-off-by: Matt Lord --- go/mysql/collations/local.go | 4 ++-- go/vt/vttablet/tabletmanager/tm_init_test.go | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/go/mysql/collations/local.go b/go/mysql/collations/local.go index 34f13b6db60..fccbed6cde8 100644 --- a/go/mysql/collations/local.go +++ b/go/mysql/collations/local.go @@ -20,7 +20,6 @@ package collations import ( "flag" - "strings" "sync" "vitess.io/vitess/go/vt/servenv" @@ -42,7 +41,8 @@ func Local() *Environment { // for the vtgate mysql_server_version flag. defaultEnv = fetchCacheEnvironment(collverMySQL57) } else { - defaultEnv = NewEnvironment(strings.Clone(*servenv.MySQLServerVersion)) + //defaultEnv = NewEnvironment(strings.Clone(*servenv.MySQLServerVersion)) + defaultEnv = NewEnvironment(*servenv.MySQLServerVersion) } }) return defaultEnv diff --git a/go/vt/vttablet/tabletmanager/tm_init_test.go b/go/vt/vttablet/tabletmanager/tm_init_test.go index 90a138977c6..cccdcb91d70 100644 --- a/go/vt/vttablet/tabletmanager/tm_init_test.go +++ b/go/vt/vttablet/tabletmanager/tm_init_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/sync2" @@ -42,6 +43,12 @@ import ( vschemapb "vitess.io/vitess/go/vt/proto/vschema" ) +var ( + dbServerVersion = "5.7.0" + charsetName = "utf8mb4" + dbsvCollID = collations.NewEnvironment(dbServerVersion).DefaultCollationForCharset(charsetName).ID() +) + func TestStartBuildTabletFromInput(t *testing.T) { alias := &topodatapb.TabletAlias{ Cell: "cell", @@ -49,7 +56,6 @@ func TestStartBuildTabletFromInput(t *testing.T) { } port := int32(12) grpcport := int32(34) - dbServerVersion := "5.7.0" // Hostname should be used as is. *tabletHostname = "foo" @@ -71,7 +77,7 @@ func TestStartBuildTabletFromInput(t *testing.T) { Tags: map[string]string{}, DbNameOverride: "aa", DbServerVersion: dbServerVersion, - DefaultConnCollation: 255, + DefaultConnCollation: uint32(dbsvCollID), } gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil) @@ -131,7 +137,6 @@ func TestBuildTabletFromInputWithBuildTags(t *testing.T) { } port := int32(12) grpcport := int32(34) - dbServerVersion := "5.7.0" // Hostname should be used as is. *tabletHostname = "foo" @@ -154,8 +159,8 @@ func TestBuildTabletFromInputWithBuildTags(t *testing.T) { Type: topodatapb.TabletType_REPLICA, Tags: servenv.AppVersion.ToStringMap(), DbNameOverride: "aa", - DbServerVersion: "5.7.0", - DefaultConnCollation: 255, + DbServerVersion: dbServerVersion, + DefaultConnCollation: uint32(dbsvCollID), } gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil) From ef719d6cb4268657aa26fb6fea8eb46dc40fb1a6 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 20:30:19 -0400 Subject: [PATCH 10/12] Fix more unit tests Signed-off-by: Matt Lord --- go/mysql/collations/local.go | 1 - go/vt/vtgate/engine/memory_sort_test.go | 8 ++++++++ go/vt/vtgate/engine/ordered_aggregate_test.go | 16 +++++++++++++--- go/vt/vtgate/engine/route_test.go | 8 ++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/go/mysql/collations/local.go b/go/mysql/collations/local.go index fccbed6cde8..7c3b2ab0b15 100644 --- a/go/mysql/collations/local.go +++ b/go/mysql/collations/local.go @@ -41,7 +41,6 @@ func Local() *Environment { // for the vtgate mysql_server_version flag. defaultEnv = fetchCacheEnvironment(collverMySQL57) } else { - //defaultEnv = NewEnvironment(strings.Clone(*servenv.MySQLServerVersion)) defaultEnv = NewEnvironment(*servenv.MySQLServerVersion) } }) diff --git a/go/vt/vtgate/engine/memory_sort_test.go b/go/vt/vtgate/engine/memory_sort_test.go index f24b968e378..80b75b61774 100644 --- a/go/vt/vtgate/engine/memory_sort_test.go +++ b/go/vt/vtgate/engine/memory_sort_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/mysql/collations" @@ -31,6 +32,13 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" ) +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func TestMemorySortExecute(t *testing.T) { fields := sqltypes.MakeTestFields( "c1|c2", diff --git a/go/vt/vtgate/engine/ordered_aggregate_test.go b/go/vt/vtgate/engine/ordered_aggregate_test.go index 6e288cf6d56..461df177963 100644 --- a/go/vt/vtgate/engine/ordered_aggregate_test.go +++ b/go/vt/vtgate/engine/ordered_aggregate_test.go @@ -31,8 +31,18 @@ import ( binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/servenv" ) +var collationEnv *collations.Environment + +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func TestOrderedAggregateExecute(t *testing.T) { assert := assert.New(t) fields := sqltypes.MakeTestFields( @@ -999,7 +1009,7 @@ func TestOrderedAggregateCollate(t *testing.T) { )}, } - collationID, _ := collations.Local().LookupID("utf8mb4_0900_ai_ci") + collationID, _ := collationEnv.LookupID("utf8mb4_0900_ai_ci") oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ Opcode: AggregateSum, @@ -1042,7 +1052,7 @@ func TestOrderedAggregateCollateAS(t *testing.T) { )}, } - collationID, _ := collations.Local().LookupID("utf8mb4_0900_as_ci") + collationID, _ := collationEnv.LookupID("utf8mb4_0900_as_ci") oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ Opcode: AggregateSum, @@ -1087,7 +1097,7 @@ func TestOrderedAggregateCollateKS(t *testing.T) { )}, } - collationID, _ := collations.Local().LookupID("utf8mb4_ja_0900_as_cs_ks") + collationID, _ := collationEnv.LookupID("utf8mb4_ja_0900_as_cs_ks") oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ Opcode: AggregateSum, diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index 569bb8d4c05..e7d58d66cd3 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/mysql/collations" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/sqlparser" @@ -49,6 +50,13 @@ var defaultSelectResult = sqltypes.MakeTestResult( "1", ) +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func TestSelectUnsharded(t *testing.T) { sel := NewRoute( Unsharded, From a0d0c3845b0359e59766c5e10fc3473accb3b44f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 16 Jul 2022 21:19:36 -0400 Subject: [PATCH 11/12] Another test fix Signed-off-by: Matt Lord --- go/mysql/collations/integration/collations_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/go/mysql/collations/integration/collations_test.go b/go/mysql/collations/integration/collations_test.go index 89dd4ace2a4..e91719090bc 100644 --- a/go/mysql/collations/integration/collations_test.go +++ b/go/mysql/collations/integration/collations_test.go @@ -35,9 +35,19 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/remote" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/sqlparser" ) +var collationEnv *collations.Environment + +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func getSQLQueries(t *testing.T, testfile string) []string { tf, err := os.Open(testfile) if err != nil { @@ -105,7 +115,7 @@ func parseWeightString(b []byte) []byte { } func (u *uca900CollationTest) Test(t *testing.T, result *sqltypes.Result) { - coll := collations.Local().LookupByName(u.collation) + coll := collationEnv.LookupByName(u.collation) if coll == nil { t.Fatalf("unknown collation %q", u.collation) } @@ -215,7 +225,7 @@ func TestCollationWithSpace(t *testing.T) { for _, collName := range []string{"utf8mb4_0900_ai_ci", "utf8mb4_unicode_ci", "utf8mb4_unicode_520_ci"} { t.Run(collName, func(t *testing.T) { - local := collations.Local().LookupByName(collName) + local := collationEnv.LookupByName(collName) remote := remote.NewCollation(conn, collName) for _, size := range []int{0, codepoints, codepoints + 1, codepoints + 2, 20, 32} { From e195cd2e1e93c06f5db0d7307d6f70c9ae8d8ecf Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 17 Jul 2022 01:00:09 -0400 Subject: [PATCH 12/12] Fix another test that requires 8.0 collations Signed-off-by: Matt Lord --- .../vtgate/evalengine/integration/comparison_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/go/vt/vtgate/evalengine/integration/comparison_test.go b/go/vt/vtgate/evalengine/integration/comparison_test.go index f55d5881aec..cecf7dc4a42 100644 --- a/go/vt/vtgate/evalengine/integration/comparison_test.go +++ b/go/vt/vtgate/evalengine/integration/comparison_test.go @@ -28,9 +28,19 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vtgate/evalengine" ) +var collationEnv *collations.Environment + +func init() { + // We require MySQL 8.0 collations for the comparisons in the tests + mySQLVersion := "8.0.0" + servenv.MySQLServerVersion = &mySQLVersion + collationEnv = collations.NewEnvironment(mySQLVersion) +} + func perm(a []string, f func([]string)) { perm1(a, f, 0) } @@ -126,7 +136,7 @@ func compareRemoteExpr(t *testing.T, conn *mysql.Conn, expr string) { // TODO: passthrough proper collations for nullable fields remoteCollation = collations.CollationBinaryID } else { - remoteCollation = collations.Local().LookupByName(remote.Rows[0][1].ToString()).ID() + remoteCollation = collationEnv.LookupByName(remote.Rows[0][1].ToString()).ID() } } }