Skip to content
This repository has been archived by the owner on Nov 24, 2023. It is now read-only.

worker: generate index flavor automatically by querying version info from DB #323

Merged
merged 20 commits into from
Oct 21, 2019
Merged

worker: generate index flavor automatically by querying version info from DB #323

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
b58e467
simplify flavor
lichunzhu Oct 17, 2019
58d3de2
fix bugs
lichunzhu Oct 17, 2019
9296fd1
remove flavor
lichunzhu Oct 17, 2019
19ac343
add decrypt password
lichunzhu Oct 17, 2019
5e14b93
Merge branch 'master' into czli/dm-worker/simplifyFlavor
lichunzhu Oct 17, 2019
0becf14
fix bug
lichunzhu Oct 17, 2019
0d01ec0
merge master
lichunzhu Oct 17, 2019
6cdbdc0
Merge branch 'master' of https://github.com/pingcap/dm into czli/dm-w…
lichunzhu Oct 18, 2019
647b0d7
move get flavor to public pkg and change createDB
lichunzhu Oct 18, 2019
c945762
refine scope and imports
lichunzhu Oct 18, 2019
310af54
Merge branch 'master' into czli/dm-worker/simplifyFlavor
IANTHEREAL Oct 19, 2019
dfb6046
use utils from tidb-tools package
lichunzhu Oct 21, 2019
e9026c5
Merge branch 'czli/dm-worker/simplifyFlavor' of https://github.com/li…
lichunzhu Oct 21, 2019
332e670
refine code
lichunzhu Oct 21, 2019
2829d5b
refine config content
lichunzhu Oct 21, 2019
7e5c1b6
Merge branch 'master' into czli/dm-worker/simplifyFlavor
IANTHEREAL Oct 21, 2019
fb7847f
add timeout for getting flavor
lichunzhu Oct 21, 2019
ada458d
refine cancel and erros msg
lichunzhu Oct 21, 2019
04b1cdb
switch to Annotatef
lichunzhu Oct 21, 2019
bc2eecf
Update dm/worker/config.go
lichunzhu Oct 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion dm/worker/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ import (
"strings"

"github.com/BurntSushi/toml"
"github.com/siddontang/go-mysql/mysql"

"github.com/pingcap/dm/dm/config"
"github.com/pingcap/dm/pkg/binlog"
"github.com/pingcap/dm/pkg/conn"
"github.com/pingcap/dm/pkg/gtid"
"github.com/pingcap/dm/pkg/log"
"github.com/pingcap/dm/pkg/terror"
Expand All @@ -39,6 +41,8 @@ import (
// and assign it to SampleConfigFile while we build dm-worker
var SampleConfigFile string

var applyNewBaseDB = conn.DefaultDBProvider.Apply

// NewConfig creates a new base config for worker.
func NewConfig() *Config {
cfg := &Config{}
Expand Down Expand Up @@ -188,6 +192,10 @@ func (c *Config) Parse(arguments []string) error {

c.From.Adjust()
c.Checker.adjust()
err = c.adjustFlavor()
if err != nil {
return err
}
return c.verify()
}

Expand Down Expand Up @@ -239,6 +247,29 @@ func (c *Config) configFromFile(path string) error {
return c.verify()
}

// adjustFlavor adjusts flavor through querying from given database
func (c *Config) adjustFlavor() error {
if c.Flavor != "" {
if c.Flavor != mysql.MariaDBFlavor && c.Flavor != mysql.MySQLFlavor {
lichunzhu marked this conversation as resolved.
Show resolved Hide resolved
return terror.ErrNotSupportedFlavor.Generate(c.Flavor)
}
return nil
}
// decrypt password
clone, err := c.DecryptPassword()
if err != nil {
return err
}
fromDB, err := applyNewBaseDB(clone.From)
if err != nil {
return terror.WithScope(err, terror.ScopeUpstream)
}
defer fromDB.Close()

c.Flavor, err = utils.GetFlavor(fromDB.DB)
csuzhangxc marked this conversation as resolved.
Show resolved Hide resolved
return terror.WithScope(err, terror.ScopeUpstream)
}

// UpdateConfigFile write configure to local file
func (c *Config) UpdateConfigFile(content string) error {
if c.ConfigFile == "" {
Expand Down Expand Up @@ -279,6 +310,5 @@ func (c *Config) DecryptPassword() (*Config, error) {
}
}
clone.From.Password = pswdFrom

return clone, nil
}
45 changes: 45 additions & 0 deletions dm/worker/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@ import (
"path"
"strings"

"github.com/DATA-DOG/go-sqlmock"
. "github.com/pingcap/check"
"github.com/siddontang/go-mysql/mysql"

"github.com/pingcap/dm/dm/config"
"github.com/pingcap/dm/pkg/conn"
)

func (t *testServer) TestConfig(c *C) {
Expand Down Expand Up @@ -160,3 +163,45 @@ func (t *testServer) TestConfigVerify(c *C) {
}

}

func subtestFlavor(c *C, cfg *Config, sqlInfo, expectedFlavor, expectedError string) {
cfg.Flavor = ""
db, mock, err := sqlmock.New()
c.Assert(err, IsNil)
mock.ExpectQuery("SELECT @@version_comment").
WillReturnRows(sqlmock.NewRows([]string{"@@version_comment"}).
AddRow(sqlInfo))
mock.ExpectClose()
applyNewBaseDB = func(config config.DBConfig) (*conn.BaseDB, error) {
return &conn.BaseDB{DB: db}, nil
}
err = cfg.adjustFlavor()
if expectedError == "" {
c.Assert(err, IsNil)
c.Assert(cfg.Flavor, Equals, expectedFlavor)
} else {
c.Assert(err, ErrorMatches, expectedError)
}
}

func (t *testServer) TestAdjustFlavor(c *C) {
cfg := NewConfig()
c.Assert(cfg.Parse([]string{"-config=./dm-worker.toml", "-relay-dir=./xx"}), IsNil)

cfg.Flavor = "mariadb"
err := cfg.adjustFlavor()
c.Assert(err, IsNil)
c.Assert(cfg.Flavor, Equals, mysql.MariaDBFlavor)
cfg.Flavor = "MongoDB"
err = cfg.adjustFlavor()
c.Assert(err, ErrorMatches, ".*flavor MongoDB not supported")

var origApplyNewBaseDB = applyNewBaseDB
defer func() {
applyNewBaseDB = origApplyNewBaseDB
}()

subtestFlavor(c, cfg, "mariadb.org binary distribution", mysql.MariaDBFlavor, "")
subtestFlavor(c, cfg, "MySQL Community Server - GPL", mysql.MySQLFlavor, "")
subtestFlavor(c, cfg, "MongoDB", "", ".*flavor MongoDB not supported")
}
20 changes: 20 additions & 0 deletions pkg/utils/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,26 @@ var (
domainServerIDSeparator = "-"
)

// GetFlavor gets flavor from DB
func GetFlavor(db *sql.DB) (string, error) {
query := "SELECT @@version_comment"
row := db.QueryRow(query)
lichunzhu marked this conversation as resolved.
Show resolved Hide resolved
var versionComment string
err := row.Scan(&versionComment)
if err != nil {
return "", terror.DBErrorAdapt(err, terror.ErrDBDriverError)
}
lowercaseVersionComment := strings.ToLower(versionComment)
switch {
case strings.Contains(lowercaseVersionComment, gmysql.MariaDBFlavor):
return gmysql.MariaDBFlavor, nil
case strings.Contains(lowercaseVersionComment, gmysql.MySQLFlavor):
return gmysql.MySQLFlavor, nil
default:
return "", terror.ErrNotSupportedFlavor.Generate(versionComment)
lichunzhu marked this conversation as resolved.
Show resolved Hide resolved
}
}

// GetMasterStatus gets status from master
func GetMasterStatus(db *sql.DB, flavor string) (gmysql.Position, gtid.Set, error) {
var (
Expand Down
2 changes: 1 addition & 1 deletion tests/all_mode/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/all_mode/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/compatibility/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/compatibility/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/dmctl_basic/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/dmctl_basic/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/http_apis/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/incremental_mode/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/incremental_mode/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/initial_unit/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/load_interrupt/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/load_interrupt/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/online_ddl/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/online_ddl/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/print_status/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/relay_interrupt/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/safe_mode/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/safe_mode/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/sequence_safe_mode/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/sequence_safe_mode/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/sequence_sharding/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/sequence_sharding/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/sharding/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/sharding/conf/dm-worker2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 102
source-id = "mysql-replica-02"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down
2 changes: 1 addition & 1 deletion tests/start_task/conf/dm-worker1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

server-id = 101
source-id = "mysql-replica-01"
flavor = "mysql"
flavor = ""
enable-gtid = false
relay-binlog-name = ""
relay-binlog-gtid = ""
Expand Down