worker: generate index flavor automatically by querying version info …

…from DB (pingcap#323)

use dbutil.ShowVersion and check.IsMariaDB from tidb-tools to detect upstream MySQL/MariaDB server type and set flavor automatically.

dm/worker/config.go

@@ -15,17 +15,21 @@ package worker
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"strings"
+	"time"
 
 	"github.com/BurntSushi/toml"
+	"github.com/siddontang/go-mysql/mysql"
 
 	"github.com/pingcap/dm/dm/config"
 	"github.com/pingcap/dm/pkg/binlog"
+	"github.com/pingcap/dm/pkg/conn"
 	"github.com/pingcap/dm/pkg/gtid"
 	"github.com/pingcap/dm/pkg/log"
 	"github.com/pingcap/dm/pkg/terror"
@@ -34,11 +38,20 @@ import (
 	"github.com/pingcap/dm/relay/purger"
 )
 
+const (
+	// flavorReadTimeout is readTimeout for DB connection in adjustFlavor
+	flavorReadTimeout = "30s"
+	// flavorGetTimeout is timeout for getting version info from DB
+	flavorGetTimeout = 30 * time.Second
+)
+
 // SampleConfigFile is sample config file of dm-worker
 // later we can read it from dm/worker/dm-worker.toml
 // and assign it to SampleConfigFile while we build dm-worker
 var SampleConfigFile string
 
+var applyNewBaseDB = conn.DefaultDBProvider.Apply
+
 // NewConfig creates a new base config for worker.
 func NewConfig() *Config {
 	cfg := &Config{}
@@ -188,6 +201,10 @@ func (c *Config) Parse(arguments []string) error {
 
 	c.From.Adjust()
 	c.Checker.adjust()
+	err = c.adjustFlavor()
+	if err != nil {
+		return err
+	}
 	return c.verify()
 }
 
@@ -239,6 +256,38 @@ func (c *Config) configFromFile(path string) error {
 	return c.verify()
 }
 
+// adjustFlavor adjusts flavor through querying from given database
+func (c *Config) adjustFlavor() error {
+	if c.Flavor != "" {
+		switch c.Flavor {
+		case mysql.MariaDBFlavor, mysql.MySQLFlavor:
+			return nil
+		default:
+			return terror.ErrNotSupportedFlavor.Generate(c.Flavor)
+		}
+	}
+	// decrypt password
+	clone, err := c.DecryptPassword()
+	if err != nil {
+		return err
+	}
+	from := clone.From
+	from.RawDBCfg = config.DefaultRawDBConfig().SetReadTimeout(flavorReadTimeout)
+	fromDB, err := applyNewBaseDB(from)
+	if err != nil {
+		return terror.WithScope(err, terror.ScopeUpstream)
+	}
+	defer fromDB.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), flavorGetTimeout)
+	defer cancel()
+	c.Flavor, err = utils.GetFlavor(ctx, fromDB.DB)
+	if ctx.Err() != nil {
+		err = terror.Annotatef(err, "time cost to get flavor info exceeds %s", flavorGetTimeout)
+	}
+	return terror.WithScope(err, terror.ScopeUpstream)
+}
+
 // UpdateConfigFile write configure to local file
 func (c *Config) UpdateConfigFile(content string) error {
 	if c.ConfigFile == "" {
@@ -279,6 +328,5 @@ func (c *Config) DecryptPassword() (*Config, error) {
 		}
 	}
 	clone.From.Password = pswdFrom
-
 	return clone, nil
 }

dm/worker/config_test.go

@@ -19,9 +19,12 @@ import (
 	"path"
 	"strings"
 
+	"github.com/DATA-DOG/go-sqlmock"
 	. "github.com/pingcap/check"
+	"github.com/siddontang/go-mysql/mysql"
 
 	"github.com/pingcap/dm/dm/config"
+	"github.com/pingcap/dm/pkg/conn"
 )
 
 func (t *testServer) TestConfig(c *C) {
@@ -160,3 +163,44 @@ func (t *testServer) TestConfigVerify(c *C) {
 	}
 
 }
+
+func subtestFlavor(c *C, cfg *Config, sqlInfo, expectedFlavor, expectedError string) {
+	cfg.Flavor = ""
+	db, mock, err := sqlmock.New()
+	c.Assert(err, IsNil)
+	mock.ExpectQuery("SHOW GLOBAL VARIABLES LIKE 'version';").
+		WillReturnRows(sqlmock.NewRows([]string{"Variable_name", "Value"}).
+			AddRow("version", sqlInfo))
+	mock.ExpectClose()
+	applyNewBaseDB = func(config config.DBConfig) (*conn.BaseDB, error) {
+		return &conn.BaseDB{DB: db}, nil
+	}
+	err = cfg.adjustFlavor()
+	if expectedError == "" {
+		c.Assert(err, IsNil)
+		c.Assert(cfg.Flavor, Equals, expectedFlavor)
+	} else {
+		c.Assert(err, ErrorMatches, expectedError)
+	}
+}
+
+func (t *testServer) TestAdjustFlavor(c *C) {
+	cfg := NewConfig()
+	c.Assert(cfg.Parse([]string{"-config=./dm-worker.toml", "-relay-dir=./xx"}), IsNil)
+
+	cfg.Flavor = "mariadb"
+	err := cfg.adjustFlavor()
+	c.Assert(err, IsNil)
+	c.Assert(cfg.Flavor, Equals, mysql.MariaDBFlavor)
+	cfg.Flavor = "MongoDB"
+	err = cfg.adjustFlavor()
+	c.Assert(err, ErrorMatches, ".*flavor MongoDB not supported")
+
+	var origApplyNewBaseDB = applyNewBaseDB
+	defer func() {
+		applyNewBaseDB = origApplyNewBaseDB
+	}()
+
+	subtestFlavor(c, cfg, "10.4.8-MariaDB-1:10.4.8+maria~bionic", mysql.MariaDBFlavor, "")
+	subtestFlavor(c, cfg, "5.7.26-log", mysql.MySQLFlavor, "")
+}

pkg/utils/db.go

@@ -14,6 +14,7 @@
 package utils
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"strconv"
@@ -28,6 +29,8 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser"
 	tmysql "github.com/pingcap/parser/mysql"
+	"github.com/pingcap/tidb-tools/pkg/check"
+	"github.com/pingcap/tidb-tools/pkg/dbutil"
 	gmysql "github.com/siddontang/go-mysql/mysql"
 	"go.uber.org/zap"
 )
@@ -37,6 +40,18 @@ var (
 	domainServerIDSeparator = "-"
 )
 
+// GetFlavor gets flavor from DB
+func GetFlavor(ctx context.Context, db *sql.DB) (string, error) {
+	value, err := dbutil.ShowVersion(ctx, db)
+	if err != nil {
+		return "", terror.DBErrorAdapt(err, terror.ErrDBDriverError)
+	}
+	if check.IsMariaDB(value) {
+		return gmysql.MariaDBFlavor, nil
+	}
+	return gmysql.MySQLFlavor, nil
+}
+
 // GetMasterStatus gets status from master
 func GetMasterStatus(db *sql.DB, flavor string) (gmysql.Position, gtid.Set, error) {
 	var (

tests/all_mode/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/all_mode/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/compatibility/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/compatibility/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/dmctl_basic/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/dmctl_basic/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/http_apis/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/incremental_mode/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/incremental_mode/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/initial_unit/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/load_interrupt/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/load_interrupt/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/online_ddl/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/online_ddl/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/print_status/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/relay_interrupt/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/safe_mode/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/safe_mode/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/sequence_safe_mode/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/sequence_safe_mode/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/sequence_sharding/conf/dm-worker1.toml

@@ -2,7 +2,7 @@
 
 server-id = 101
 source-id = "mysql-replica-01"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""

tests/sequence_sharding/conf/dm-worker2.toml

@@ -2,7 +2,7 @@
 
 server-id = 102
 source-id = "mysql-replica-02"
-flavor = "mysql"
+flavor = ""
 enable-gtid = false
 relay-binlog-name = ""
 relay-binlog-gtid = ""