[release-14.0] Auto Detect MySQL Version and Use in vtgate mysql_serv…

…er_version Flag (#10733)

* Auto Detect MySQL Version and Use in vtgate mysql_server_version Flag (#10701)

* Auto detect MySQL version and use in vtgate --mysql_server_version flag

Signed-off-by: Matt Lord <[email protected]>

* Remove any explicit static usage of mysql_server_version

Signed-off-by: Matt Lord <[email protected]>

* The reservedconns test needs to override the mysql_server_version

Signed-off-by: Matt Lord <[email protected]>

* Explicitly allow overriding the mysql_system_version

Signed-off-by: Matt Lord <[email protected]>

* minimize diff

Signed-off-by: Matt Lord <[email protected]>

* Fix #10712

Signed-off-by: Matt Lord <[email protected]>

* Use more robust string comparisons

Signed-off-by: Matt Lord <[email protected]>

* Fix collation handling in tests

Signed-off-by: Matt Lord <[email protected]>

* Fix tm_init_test

Signed-off-by: Matt Lord <[email protected]>

* Fix more unit tests

Signed-off-by: Matt Lord <[email protected]>

* Another test fix

Signed-off-by: Matt Lord <[email protected]>

* Fix another test that requires 8.0 collations

Signed-off-by: Matt Lord <[email protected]>

* Kick the CI :-|

Signed-off-by: Matt Lord <[email protected]>

go/mysql/collations/env.go

@@ -125,13 +125,17 @@ func fetchCacheEnvironment(version collver) *Environment {
 // The version string must be in the format that is sent by the server as the version packet
 // when opening a new MySQL connection
 func NewEnvironment(serverVersion string) *Environment {
-	var version collver = collverMySQL56
+	// 5.7 is the oldest version we support today, so use that as
+	// the default.
+	// NOTE: this should be changed when we EOL MySQL 5.7 support
+	var version collver = collverMySQL57
+	serverVersion = strings.TrimSpace(strings.ToLower(serverVersion))
 	switch {
 	case strings.HasSuffix(serverVersion, "-ripple"):
 		// the ripple binlog server can mask the actual version of mysqld;
 		// assume we have the highest
 		version = collverMySQL80
-	case strings.Contains(serverVersion, "MariaDB"):
+	case strings.Contains(serverVersion, "mariadb"):
 		switch {
 		case strings.Contains(serverVersion, "10.0."):
 			version = collverMariaDB100

go/mysql/collations/integration/collations_test.go

@@ -35,9 +35,19 @@ import (
 	"vitess.io/vitess/go/mysql/collations"
 	"vitess.io/vitess/go/mysql/collations/remote"
 	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
+var collationEnv *collations.Environment
+
+func init() {
+	// We require MySQL 8.0 collations for the comparisons in the tests
+	mySQLVersion := "8.0.0"
+	servenv.MySQLServerVersion = &mySQLVersion
+	collationEnv = collations.NewEnvironment(mySQLVersion)
+}
+
 func getSQLQueries(t *testing.T, testfile string) []string {
 	tf, err := os.Open(testfile)
 	if err != nil {
@@ -105,7 +115,7 @@ func parseWeightString(b []byte) []byte {
 }
 
 func (u *uca900CollationTest) Test(t *testing.T, result *sqltypes.Result) {
-	coll := collations.Local().LookupByName(u.collation)
+	coll := collationEnv.LookupByName(u.collation)
 	if coll == nil {
 		t.Fatalf("unknown collation %q", u.collation)
 	}
@@ -215,7 +225,7 @@ func TestCollationWithSpace(t *testing.T) {
 
 	for _, collName := range []string{"utf8mb4_0900_ai_ci", "utf8mb4_unicode_ci", "utf8mb4_unicode_520_ci"} {
 		t.Run(collName, func(t *testing.T) {
-			local := collations.Local().LookupByName(collName)
+			local := collationEnv.LookupByName(collName)
 			remote := remote.NewCollation(conn, collName)
 
 			for _, size := range []int{0, codepoints, codepoints + 1, codepoints + 2, 20, 32} {

go/mysql/collations/local.go

@@ -36,7 +36,10 @@ func Local() *Environment {
 			panic("collations.Local() called too early")
 		}
 		if *servenv.MySQLServerVersion == "" {
-			defaultEnv = fetchCacheEnvironment(collverMySQL80)
+			// The default server version used by vtgate is 5.7.9
+			// NOTE: this should be changed along with the effective default
+			//       for the vtgate mysql_server_version flag.
+			defaultEnv = fetchCacheEnvironment(collverMySQL57)
 		} else {
 			defaultEnv = NewEnvironment(*servenv.MySQLServerVersion)
 		}

go/mysql/endtoend/query_test.go

@@ -33,6 +33,10 @@ import (
 	querypb "vitess.io/vitess/go/vt/proto/query"
 )
 
+const (
+	charsetName = "utf8mb4"
+)
+
 func columnSize(cs collations.ID, size uint32) uint32 {
 	// utf8_general_ci results in smaller max column sizes because MySQL 5.7 is silly
 	if collations.Local().LookupByID(cs).Charset().Name() == "utf8mb3" {
@@ -78,6 +82,7 @@ func TestQueries(t *testing.T) {
 	if err != nil {
 		t.Fatalf("insert failed: %v", err)
 	}
+	collID := getDefaultCollationID()
 	expectedResult := &sqltypes.Result{
 		Fields: []*querypb.Field{
 			{
@@ -101,8 +106,8 @@ func TestQueries(t *testing.T) {
 				OrgTable:     "a",
 				Database:     "vttest",
 				OrgName:      "name",
-				ColumnLength: columnSize(conn.CharacterSet, 512),
-				Charset:      uint32(conn.CharacterSet),
+				ColumnLength: columnSize(collID, 512),
+				Charset:      uint32(collID),
 			},
 		},
 		Rows: [][]sqltypes.Value{
@@ -188,6 +193,7 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) {
 	}
 
 	// Check the fields.
+	collID := getDefaultCollationID()
 	expectedFields := []*querypb.Field{
 		{
 			Name:         "id",
@@ -210,8 +216,8 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) {
 			OrgTable:     "a",
 			Database:     "vttest",
 			OrgName:      "name",
-			ColumnLength: columnSize(conn.CharacterSet, 512),
-			Charset:      uint32(conn.CharacterSet),
+			ColumnLength: columnSize(collID, 512),
+			Charset:      uint32(collID),
 		},
 	}
 	fields, err := conn.Fields()
@@ -306,7 +312,8 @@ func TestSysInfo(t *testing.T) {
 	_, err = conn.ExecuteFetch("drop table if exists `a`", 1000, true)
 	require.NoError(t, err)
 
-	_, err = conn.ExecuteFetch("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY (`one`,`two`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", 1000, true)
+	_, err = conn.ExecuteFetch(fmt.Sprintf("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY (`one`,`two`)) ENGINE=InnoDB DEFAULT CHARSET=%s",
+		charsetName), 1000, true)
 	require.NoError(t, err)
 	defer conn.ExecuteFetch("drop table `a`", 1000, true)
 
@@ -339,3 +346,9 @@ func TestSysInfo(t *testing.T) {
 	assert.EqualValues(t, sqltypes.Uint64, qr.Fields[4].Type)
 	assert.EqualValues(t, querypb.Type_UINT64, qr.Rows[0][4].Type())
 }
+
+func getDefaultCollationID() collations.ID {
+	collationHandler := collations.Local()
+	collation := collationHandler.DefaultCollationForCharset(charsetName)
+	return collation.ID()
+}

go/test/endtoend/cluster/vtgate_process.go

@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"vitess.io/vitess/go/vt/log"
+	"vitess.io/vitess/go/vt/mysqlctl"
 	"vitess.io/vitess/go/vt/vtgate/planbuilder"
 	"vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext"
 )
@@ -69,7 +70,6 @@ const defaultVtGatePlannerVersion = planbuilder.Gen4CompareV3
 
 // Setup starts Vtgate process with required arguements
 func (vtgate *VtgateProcess) Setup() (err error) {
-
 	args := []string{
 		"--topo_implementation", vtgate.CommonArg.TopoImplementation,
 		"--topo_global_server_address", vtgate.CommonArg.TopoGlobalAddress,
@@ -86,6 +86,28 @@ func (vtgate *VtgateProcess) Setup() (err error) {
 		"--service_map", vtgate.ServiceMap,
 		"--mysql_auth_server_impl", vtgate.MySQLAuthServerImpl,
 	}
+	// If no explicit mysql_server_version has been specified then we autodetect
+	// the MySQL version that will be used for the test and base the vtgate's
+	// mysql server version on that.
+	msvflag := false
+	for _, f := range vtgate.ExtraArgs {
+		if strings.Contains(f, "mysql_server_version") {
+			msvflag = true
+			break
+		}
+	}
+	if !msvflag {
+		version, err := mysqlctl.GetVersionString()
+		if err != nil {
+			return err
+		}
+		_, vers, err := mysqlctl.ParseVersionString(version)
+		if err != nil {
+			return err
+		}
+		mysqlvers := fmt.Sprintf("%d.%d.%d-vitess", vers.Major, vers.Minor, vers.Patch)
+		args = append(args, "--mysql_server_version", mysqlvers)
+	}
 	if vtgate.PlannerVersion > 0 {
 		args = append(args, "--planner-version", vtgate.PlannerVersion.String())
 	}

go/test/endtoend/vtgate/reservedconn/main_test.go

@@ -124,6 +124,8 @@ func TestMain(m *testing.M) {
 		}
 
 		// Start vtgate
+		// This test requires setting the mysql_server_version vtgate flag
+		// to 5.7 regardless of the actual MySQL version used for the tests.
 		clusterInstance.VtGateExtraArgs = []string{"--lock_heartbeat_time", "2s", "--mysql_server_version", "5.7.0"}
 		if err := clusterInstance.StartVtgate(); err != nil {
 			return 1

go/vt/vtgate/engine/memory_sort_test.go

@@ -19,6 +19,7 @@ package engine
 import (
 	"testing"
 
+	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/vtgate/evalengine"
 
 	"vitess.io/vitess/go/mysql/collations"
@@ -30,6 +31,13 @@ import (
 	querypb "vitess.io/vitess/go/vt/proto/query"
 )
 
+func init() {
+	// We require MySQL 8.0 collations for the comparisons in the tests
+	mySQLVersion := "8.0.0"
+	servenv.MySQLServerVersion = &mySQLVersion
+	collationEnv = collations.NewEnvironment(mySQLVersion)
+}
+
 func TestMemorySortExecute(t *testing.T) {
 	fields := sqltypes.MakeTestFields(
 		"c1|c2",

go/vt/vtgate/engine/ordered_aggregate_test.go

@@ -30,8 +30,18 @@ import (
 
 	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
 	querypb "vitess.io/vitess/go/vt/proto/query"
+	"vitess.io/vitess/go/vt/servenv"
 )
 
+var collationEnv *collations.Environment
+
+func init() {
+	// We require MySQL 8.0 collations for the comparisons in the tests
+	mySQLVersion := "8.0.0"
+	servenv.MySQLServerVersion = &mySQLVersion
+	collationEnv = collations.NewEnvironment(mySQLVersion)
+}
+
 func TestOrderedAggregateExecute(t *testing.T) {
 	assert := assert.New(t)
 	fields := sqltypes.MakeTestFields(
@@ -998,7 +1008,7 @@ func TestOrderedAggregateCollate(t *testing.T) {
 		)},
 	}
 
-	collationID, _ := collations.Local().LookupID("utf8mb4_0900_ai_ci")
+	collationID, _ := collationEnv.LookupID("utf8mb4_0900_ai_ci")
 	oa := &OrderedAggregate{
 		Aggregates: []*AggregateParams{{
 			Opcode: AggregateSum,
@@ -1041,7 +1051,7 @@ func TestOrderedAggregateCollateAS(t *testing.T) {
 		)},
 	}
 
-	collationID, _ := collations.Local().LookupID("utf8mb4_0900_as_ci")
+	collationID, _ := collationEnv.LookupID("utf8mb4_0900_as_ci")
 	oa := &OrderedAggregate{
 		Aggregates: []*AggregateParams{{
 			Opcode: AggregateSum,
@@ -1086,7 +1096,7 @@ func TestOrderedAggregateCollateKS(t *testing.T) {
 		)},
 	}
 
-	collationID, _ := collations.Local().LookupID("utf8mb4_ja_0900_as_cs_ks")
+	collationID, _ := collationEnv.LookupID("utf8mb4_ja_0900_as_cs_ks")
 	oa := &OrderedAggregate{
 		Aggregates: []*AggregateParams{{
 			Opcode: AggregateSum,

go/vt/vtgate/engine/route_test.go

@@ -26,6 +26,7 @@ import (
 
 	"vitess.io/vitess/go/mysql/collations"
 	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
+	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/vterrors"
 
 	"vitess.io/vitess/go/vt/sqlparser"
@@ -48,6 +49,13 @@ var defaultSelectResult = sqltypes.MakeTestResult(
 	"1",
 )
 
+func init() {
+	// We require MySQL 8.0 collations for the comparisons in the tests
+	mySQLVersion := "8.0.0"
+	servenv.MySQLServerVersion = &mySQLVersion
+	collationEnv = collations.NewEnvironment(mySQLVersion)
+}
+
 func TestSelectUnsharded(t *testing.T) {
 	sel := NewRoute(
 		Unsharded,

go/vt/vtgate/evalengine/comparisons_test.go

@@ -26,6 +26,7 @@ import (
 
 	"vitess.io/vitess/go/mysql/collations"
 	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
+	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vterrors"
 
@@ -45,13 +46,21 @@ type testCase struct {
 }
 
 var (
-	T = true
-	F = false
+	T            = true
+	F            = false
+	collationEnv *collations.Environment
 )
 
+func init() {
+	// We require MySQL 8.0 collations for the comparisons in the tests
+	mySQLVersion := "8.0.0"
+	servenv.MySQLServerVersion = &mySQLVersion
+	collationEnv = collations.NewEnvironment(mySQLVersion)
+}
+
 func defaultCollation() collations.TypedCollation {
 	return collations.TypedCollation{
-		Collation:    collations.Local().LookupByName("utf8mb4_bin").ID(),
+		Collation:    collationEnv.LookupByName("utf8mb4_bin").ID(),
 		Coercibility: collations.CoerceImplicit,
 		Repertoire:   collations.RepertoireASCII,
 	}
@@ -1053,7 +1062,7 @@ func TestNullComparisons(t *testing.T) {
 }
 
 func TestNullsafeCompare(t *testing.T) {
-	collation := collations.Local().LookupByName("utf8mb4_general_ci").ID()
+	collation := collationEnv.LookupByName("utf8mb4_general_ci").ID()
 	tcases := []struct {
 		v1, v2 sqltypes.Value
 		out    int
@@ -1155,7 +1164,7 @@ func TestNullsafeCompare(t *testing.T) {
 }
 
 func getCollationID(collation string) collations.ID {
-	id, _ := collations.Local().LookupID(collation)
+	id, _ := collationEnv.LookupID(collation)
 	return id
 }
 

go/vt/vtgate/evalengine/integration/comparison_test.go

@@ -28,9 +28,19 @@ import (
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/mysql/collations"
 	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/vtgate/evalengine"
 )
 
+var collationEnv *collations.Environment
+
+func init() {
+	// We require MySQL 8.0 collations for the comparisons in the tests
+	mySQLVersion := "8.0.0"
+	servenv.MySQLServerVersion = &mySQLVersion
+	collationEnv = collations.NewEnvironment(mySQLVersion)
+}
+
 func perm(a []string, f func([]string)) {
 	perm1(a, f, 0)
 }
@@ -126,7 +136,7 @@ func compareRemoteExpr(t *testing.T, conn *mysql.Conn, expr string) {
 				// TODO: passthrough proper collations for nullable fields
 				remoteCollation = collations.CollationBinaryID
 			} else {
-				remoteCollation = collations.Local().LookupByName(remote.Rows[0][1].ToString()).ID()
+				remoteCollation = collationEnv.LookupByName(remote.Rows[0][1].ToString()).ID()
 			}
 		}
 	}