diff --git a/Makefile b/Makefile index f98453a2e1..1c04f98581 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,6 @@ LDFLAGS += -X "github.com/pingcap/dm/pkg/utils.GitHash=$(shell git rev-parse HEA LDFLAGS += -X "github.com/pingcap/dm/pkg/utils.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)" LDFLAGS += -X "github.com/pingcap/dm/pkg/utils.GoVersion=$(shell go version)" - CURDIR := $(shell pwd) GO := GO111MODULE=on go GOBUILD := CGO_ENABLED=0 $(GO) build @@ -21,10 +20,18 @@ ifeq ("$(WITH_RACE)", "1") GOBUILD = CGO_ENABLED=1 $(GO) build endif -ARCH := "`uname -s`" +ARCH := "$(shell uname -s)" LINUX := "Linux" MAC := "Darwin" +ifeq ($(ARCH), $(LINUX)) + LDFLAGS += -X "github.com/pingcap/dm/dm/worker.SampleConfigFile=$(shell cat dm/worker/dm-worker.toml | base64 -w 0)" + LDFLAGS += -X "github.com/pingcap/dm/dm/master.SampleConfigFile=$(shell cat dm/master/dm-master.toml | base64 -w 0)" +else + LDFLAGS += -X "github.com/pingcap/dm/dm/worker.SampleConfigFile=$(shell cat dm/worker/dm-worker.toml | base64)" + LDFLAGS += -X "github.com/pingcap/dm/dm/master.SampleConfigFile=$(shell cat dm/master/dm-master.toml | base64)" +endif + .PHONY: build test dm_integration_test_build integration_test coverage check \ dm-worker dm-master dmctl diff --git a/README.md b/README.md index be67f20740..44040f3b8a 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,30 @@ make dmctl # build dmctl When DM is built successfully, you can find binaries in the `bin` directory. +## Installing + +* The best way to install DM is via [DM-Ansible](https://pingcap.com/docs/tools/data-migration-deployment/) +* deploy DM manually + ``` + # Download the DM package. + wget http://download.pingcap.org/dm-latest-linux-amd64.tar.gz + wget http://download.pingcap.org/dm-latest-linux-amd64.sha256 + + # Check the file integrity. If the result is OK, the file is correct. + sha256sum -c dm-latest-linux-amd64.sha256 + + # Extract the package. + tar -xzf dm-latest-linux-amd64.tar.gz + cd dm-latest-linux-amd64 + ``` + +## Config File + +* all sample config files can be found in directory `conf` of dm tarball +* sample config file of dm-master: `bin/dm-master -print-sample-config` +* sample config file of dm-worker: `bin/dm-worker -print-sample-config` + + ## Contributing Contributions are welcomed and greatly appreciated. See [CONTRIBUTING.md](./CONTRIBUTING.md) for details on submitting patches and the contribution workflow. diff --git a/dm/master/config.go b/dm/master/config.go index ccfab02e74..c4dbe1f6c3 100644 --- a/dm/master/config.go +++ b/dm/master/config.go @@ -14,10 +14,12 @@ package master import ( + "encoding/base64" "encoding/json" "flag" "fmt" "io/ioutil" + "strings" "github.com/BurntSushi/toml" "github.com/pingcap/dm/pkg/log" @@ -25,6 +27,11 @@ import ( "github.com/pingcap/errors" ) +// SampleConfigFile is sample config file of dm-master +// later we can read it from dm/master/dm-master.toml +// and assign it to SampleConfigFile while we build dm-master +var SampleConfigFile string + // NewConfig creates a config for dm-master func NewConfig() *Config { cfg := &Config{} @@ -32,6 +39,7 @@ func NewConfig() *Config { fs := cfg.FlagSet fs.BoolVar(&cfg.printVersion, "V", false, "prints version and exit") + fs.BoolVar(&cfg.printSampleConfig, "print-sample-config", false, "print sample config file of dm-worker") fs.StringVar(&cfg.ConfigFile, "config", "", "path to config file") fs.StringVar(&cfg.MasterAddr, "master-addr", "", "master API server and status addr") fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal") @@ -72,7 +80,8 @@ type Config struct { ConfigFile string `json:"config-file"` - printVersion bool + printVersion bool + printSampleConfig bool } func (c *Config) String() string { @@ -96,6 +105,20 @@ func (c *Config) Parse(arguments []string) error { return flag.ErrHelp } + if c.printSampleConfig { + if strings.TrimSpace(SampleConfigFile) == "" { + fmt.Println("sample config file of dm-master is empty") + } else { + rawConfig, err := base64.StdEncoding.DecodeString(SampleConfigFile) + if err != nil { + fmt.Println("base64 decode config error:", err) + } else { + fmt.Println(string(rawConfig)) + } + } + return flag.ErrHelp + } + // Load config file if specified. if c.ConfigFile != "" { err = c.configFromFile(c.ConfigFile) diff --git a/dm/master/dm-master.toml b/dm/master/dm-master.toml index cd995e6f77..4937ecdfc0 100644 --- a/dm/master/dm-master.toml +++ b/dm/master/dm-master.toml @@ -1,13 +1,13 @@ # Master Configuration. +#log configuration log-level = "info" log-file = "dm-master.log" -log-rotate = "day" +#dm-master listen address master-addr = ":8261" -# mysql <-> Worker deployment, we'll refine it when new deployment function is available - +# replication group <-> dm-Worker deployment, we'll refine it when new deployment function is available [[deploy]] source-id = "mysql-replica-01" dm-worker = "172.16.10.72:8262" diff --git a/dm/worker/config.go b/dm/worker/config.go index 4a667ebd07..91c17a9410 100644 --- a/dm/worker/config.go +++ b/dm/worker/config.go @@ -15,10 +15,12 @@ package worker import ( "bytes" + "encoding/base64" "encoding/json" "flag" "fmt" "io/ioutil" + "strings" "github.com/BurntSushi/toml" "github.com/pingcap/dm/pkg/log" @@ -31,6 +33,11 @@ import ( "github.com/pingcap/dm/relay/purger" ) +// SampleConfigFile is sample config file of dm-worker +// later we can read it from dm/worker/dm-worker.toml +// and assign it to SampleConfigFile while we build dm-worker +var SampleConfigFile string + // NewConfig creates a new base config for worker. func NewConfig() *Config { cfg := &Config{} @@ -38,6 +45,7 @@ func NewConfig() *Config { fs := cfg.flagSet fs.BoolVar(&cfg.printVersion, "V", false, "prints version and exit") + fs.BoolVar(&cfg.printSampleConfig, "print-sample-config", false, "print sample config file of dm-worker") fs.StringVar(&cfg.ConfigFile, "config", "", "path to config file") fs.StringVar(&cfg.WorkerAddr, "worker-addr", "", "worker API server and status addr") fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal") @@ -63,7 +71,6 @@ type Config struct { EnableGTID bool `toml:"enable-gtid" json:"enable-gtid"` AutoFixGTID bool `toml:"auto-fix-gtid" json:"auto-fix-gtid"` - MetaFile string `toml:"meta-file" json:"meta-file"` RelayDir string `toml:"relay-dir" json:"relay-dir"` ServerID int `toml:"server-id" json:"server-id"` Flavor string `toml:"flavor" json:"flavor"` @@ -81,7 +88,8 @@ type Config struct { ConfigFile string `json:"config-file"` - printVersion bool + printVersion bool + printSampleConfig bool } // Clone clones a config @@ -141,6 +149,20 @@ func (c *Config) Parse(arguments []string) error { return flag.ErrHelp } + if c.printSampleConfig { + if strings.TrimSpace(SampleConfigFile) == "" { + fmt.Println("sample config file of dm-worker is empty") + } else { + rawConfig, err := base64.StdEncoding.DecodeString(SampleConfigFile) + if err != nil { + fmt.Println("base64 decode config error:", err) + } else { + fmt.Println(string(rawConfig)) + } + } + return flag.ErrHelp + } + // Load config file if specified. if c.ConfigFile != "" { err = c.configFromFile(c.ConfigFile) diff --git a/dm/worker/dm-worker.toml b/dm/worker/dm-worker.toml index 931d71ed46..f8b178970c 100644 --- a/dm/worker/dm-worker.toml +++ b/dm/worker/dm-worker.toml @@ -1,18 +1,29 @@ # Worker Configuration. +#log configuration log-level = "info" log-file = "dm-worker.log" -log-rotate = "day" +#dm-worker listen address worker-addr = ":8262" - +#server id of slave for binlog replication +#each instance (master and slave) in replication group should have different server id server-id = 101 -source-id = "127.0.0.1:3306" + +#represents a MySQL/MariaDB instance or a replication group +source-id = "mysql-replica-01" + +#flavor: mysql/mariadb flavor = "mysql" + +#directory that used to store relay log relay-dir = "./relay_log" -meta-file = "relay.meta" + +#enable gtid in relay log unit enable-gtid = false + +#charset of DSN of source mysql/mariadb instance # charset= "" [from] @@ -21,6 +32,7 @@ user = "root" password = "" port = 3306 +#relay log purge strategy #[purge] #interval = 3600 #expires = 24 diff --git a/dm/worker/relay.go b/dm/worker/relay.go index 34b454faed..597642487a 100644 --- a/dm/worker/relay.go +++ b/dm/worker/relay.go @@ -47,7 +47,6 @@ func NewRelayHolder(cfg *Config) *RelayHolder { EnableGTID: cfg.EnableGTID, AutoFixGTID: cfg.AutoFixGTID, Flavor: cfg.Flavor, - MetaFile: cfg.MetaFile, RelayDir: cfg.RelayDir, ServerID: cfg.ServerID, Charset: cfg.Charset, diff --git a/dm/worker/worker.go b/dm/worker/worker.go index 7023414654..d84dedefb3 100644 --- a/dm/worker/worker.go +++ b/dm/worker/worker.go @@ -203,7 +203,6 @@ func (w *Worker) copyConfigFromWorker(cfg *config.SubTaskConfig) { // log config items, mydumper unit use it cfg.LogLevel = w.cfg.LogLevel cfg.LogFile = w.cfg.LogFile - cfg.LogRotate = w.cfg.LogRotate } // StopSubTask stops a running sub task diff --git a/relay/config.go b/relay/config.go index 448deddda8..a67fba4c2f 100644 --- a/relay/config.go +++ b/relay/config.go @@ -23,7 +23,6 @@ import ( type Config struct { EnableGTID bool `toml:"enable-gtid" json:"enable-gtid"` AutoFixGTID bool `toml:"auto-fix-gtid" json:"auto-fix-gtid"` - MetaFile string `toml:"meta-file" json:"meta-file"` RelayDir string `toml:"relay-dir" json:"relay-dir"` ServerID int `toml:"server-id" json:"server-id"` Flavor string `toml:"flavor" json:"flavor"`