Merge branch 'master' into check-config-nightly

components/dm/ansible/import.go

@@ -330,8 +330,9 @@ func (im *Importer) ImportFromAnsibleDir(ctx context.Context) (clusterName strin
 		case "dm_master_servers":
 			for _, host := range group.Hosts {
 				srv := &spec.MasterSpec{
-					Host:    host.Vars["ansible_host"],
-					SSHPort: ansible.GetHostPort(host, cfg),
+					Host:     host.Vars["ansible_host"],
+					SSHPort:  ansible.GetHostPort(host, cfg),
+					Imported: true,
 				}
 
 				runFileName := filepath.Join(host.Vars["deploy_dir"], "scripts", "run_dm-master.sh")
@@ -380,6 +381,7 @@ func (im *Importer) ImportFromAnsibleDir(ctx context.Context) (clusterName strin
 					Host:      host.Vars["ansible_host"],
 					SSHPort:   ansible.GetHostPort(host, cfg),
 					DeployDir: firstNonEmpty(host.Vars["deploy_dir"], topo.GlobalOptions.DeployDir),
+					Imported:  true,
 				}
 
 				runFileName := filepath.Join(host.Vars["deploy_dir"], "scripts", "run_dm-worker.sh")
@@ -441,6 +443,7 @@ func (im *Importer) ImportFromAnsibleDir(ctx context.Context) (clusterName strin
 					Host:      host.Vars["ansible_host"],
 					SSHPort:   ansible.GetHostPort(host, cfg),
 					DeployDir: firstNonEmpty(host.Vars["deploy_dir"], topo.GlobalOptions.DeployDir),
+					Imported:  true,
 				}
 
 				runFileName := filepath.Join(host.Vars["deploy_dir"], "scripts", "run_prometheus.sh")
@@ -489,6 +492,7 @@ func (im *Importer) ImportFromAnsibleDir(ctx context.Context) (clusterName strin
 					Host:      host.Vars["ansible_host"],
 					SSHPort:   ansible.GetHostPort(host, cfg),
 					DeployDir: firstNonEmpty(host.Vars["deploy_dir"], topo.GlobalOptions.DeployDir),
+					Imported:  true,
 				}
 
 				runFileName := filepath.Join(host.Vars["deploy_dir"], "scripts", "run_alertmanager.sh")
@@ -546,6 +550,7 @@ func (im *Importer) ImportFromAnsibleDir(ctx context.Context) (clusterName strin
 					Port:     port,
 					Username: grafanaUser,
 					Password: grafanaPass,
+					Imported: true,
 				}
 
 				runFileName := filepath.Join(host.Vars["deploy_dir"], "scripts", "run_grafana.sh")

components/dm/ansible/import_test.go

@@ -164,6 +164,7 @@ func TestImportFromAnsible(t *testing.T) {
 		DeployDir: "",
 		LogDir:    "/home/tidb/deploy/log",
 		Config:    map[string]interface{}{"log-level": "info"},
+		Imported:  true,
 	}
 	assert.Equal(expectedMaster, master)
 
@@ -180,6 +181,7 @@ func TestImportFromAnsible(t *testing.T) {
 		DeployDir: "/home/tidb/deploy",
 		LogDir:    "/home/tidb/deploy/log",
 		Config:    map[string]interface{}{"log-level": "info"},
+		Imported:  true,
 	}
 
 	worker := topo.Workers[0]
@@ -199,6 +201,7 @@ func TestImportFromAnsible(t *testing.T) {
 		DeployDir: "",
 		DataDir:   "/home/tidb/deploy/data.alertmanager",
 		LogDir:    "/home/tidb/deploy/log",
+		Imported:  true,
 	}
 	assert.Equal(expectedAlter, aler)
 
@@ -212,6 +215,7 @@ func TestImportFromAnsible(t *testing.T) {
 		Port:      3001,
 		Username:  "foo",
 		Password:  "bar",
+		Imported:  true,
 	}
 	assert.Equal(expectedGrafana, grafana)
 
@@ -225,6 +229,7 @@ func TestImportFromAnsible(t *testing.T) {
 		DataDir:   "/home/tidb/deploy/prometheus.data.metrics",
 		LogDir:    "/home/tidb/deploy/log",
 		Port:      9090,
+		Imported:  true,
 	}
 	assert.Equal(expectedMonitor, monitor)
 

components/playground/instance/tiflash.go

@@ -76,8 +76,7 @@ type scheduleConfig struct {
 }
 
 type replicateMaxReplicaConfig struct {
-	MaxReplicas          int    `json:"max-replicas"`
-	EnablePlacementRules string `json:"enable-placement-rules"`
+	MaxReplicas int `json:"max-replicas"`
 }
 
 type replicateEnablePlacementRulesConfig struct {
@@ -123,8 +122,7 @@ func (inst *TiFlashInstance) Start(ctx context.Context, version utils.Version) e
 	}
 	// Update maxReplicas before placement rules so that it would not be overwritten
 	maxReplicas, err := json.Marshal(replicateMaxReplicaConfig{
-		MaxReplicas:          1,
-		EnablePlacementRules: "false",
+		MaxReplicas: 1,
 	})
 	if err != nil {
 		return err
@@ -154,7 +152,7 @@ func (inst *TiFlashInstance) Start(ctx context.Context, version utils.Version) e
 		if err != nil {
 			return err
 		}
-		if version, err = env.GetComponentInstalledVersion("tiflash", version); err != nil {
+		if version, err = env.DownloadComponentIfMissing("tiflash", version); err != nil {
 			return err
 		}
 		// version may be empty, we will use the latest stable version later in Start cmd.

embed/templates/scripts/run_pd_scale.sh.tpl

@@ -10,9 +10,9 @@ cd "${DEPLOY_DIR}" || exit 1
 {{- define "PDList"}}
   {{- range $idx, $pd := .}}
     {{- if eq $idx 0}}
-      {{- $pd.Name}}={{$pd.AdvertisePeerAddr}}
+      {{- $pd.AdvertiseClientAddr}}
     {{- else -}}
-      ,{{- $pd.Name}}={{$pd.AdvertisePeerAddr}}
+      ,{{- $pd.AdvertiseClientAddr}}
     {{- end}}
   {{- end}}
 {{- end}}

pkg/cluster/ansible/import.go

@@ -193,9 +193,9 @@ func LoadConfig(clsName string, cls *spec.ClusterMeta) error {
 		}
 	}
 	global, locals := diffConfigs(configs)
-	cls.Topology.ServerConfigs.TiDB = global
+	cls.Topology.ServerConfigs.TiDB = spec.MergeConfig(cls.Topology.ServerConfigs.TiDB, global)
 	for i, local := range locals {
-		cls.Topology.TiDBServers[i].Config = local
+		cls.Topology.TiDBServers[i].Config = spec.MergeConfig(cls.Topology.TiDBServers[i].Config, local)
 	}
 
 	// deal with tikv config
@@ -210,9 +210,9 @@ func LoadConfig(clsName string, cls *spec.ClusterMeta) error {
 		}
 	}
 	global, locals = diffConfigs(configs)
-	cls.Topology.ServerConfigs.TiKV = global
+	cls.Topology.ServerConfigs.TiKV = spec.MergeConfig(cls.Topology.ServerConfigs.TiKV, global)
 	for i, local := range locals {
-		cls.Topology.TiKVServers[i].Config = local
+		cls.Topology.TiKVServers[i].Config = spec.MergeConfig(cls.Topology.TiKVServers[i].Config, local)
 	}
 
 	// deal with pd config
@@ -227,9 +227,9 @@ func LoadConfig(clsName string, cls *spec.ClusterMeta) error {
 		}
 	}
 	global, locals = diffConfigs(configs)
-	cls.Topology.ServerConfigs.PD = global
+	cls.Topology.ServerConfigs.PD = spec.MergeConfig(cls.Topology.ServerConfigs.PD, global)
 	for i, local := range locals {
-		cls.Topology.PDServers[i].Config = local
+		cls.Topology.PDServers[i].Config = spec.MergeConfig(cls.Topology.PDServers[i].Config, local)
 	}
 
 	// deal with pump config
@@ -244,9 +244,9 @@ func LoadConfig(clsName string, cls *spec.ClusterMeta) error {
 		}
 	}
 	global, locals = diffConfigs(configs)
-	cls.Topology.ServerConfigs.Pump = global
+	cls.Topology.ServerConfigs.Pump = spec.MergeConfig(cls.Topology.ServerConfigs.Pump, global)
 	for i, local := range locals {
-		cls.Topology.PumpServers[i].Config = local
+		cls.Topology.PumpServers[i].Config = spec.MergeConfig(cls.Topology.PumpServers[i].Config, local)
 	}
 
 	// deal with drainer config
@@ -261,9 +261,9 @@ func LoadConfig(clsName string, cls *spec.ClusterMeta) error {
 		}
 	}
 	global, locals = diffConfigs(configs)
-	cls.Topology.ServerConfigs.Drainer = global
+	cls.Topology.ServerConfigs.Drainer = spec.MergeConfig(cls.Topology.ServerConfigs.Drainer, global)
 	for i, local := range locals {
-		cls.Topology.Drainers[i].Config = local
+		cls.Topology.Drainers[i].Config = spec.MergeConfig(cls.Topology.Drainers[i].Config, local)
 	}
 
 	return nil

pkg/cluster/manager/deploy.go

@@ -187,8 +187,9 @@ func (m *Manager) Deploy(
 	iterErr = nil
 	topo.IterInstance(func(inst spec.Instance) {
 		if _, found := uniqueHosts[inst.GetHost()]; !found {
-			// check for "imported" parameter, it can not be true when scaling out
-			if inst.IsImported() {
+			// check for "imported" parameter, it can not be true when deploying and scaling out
+			// only for tidb now, need to support dm
+			if inst.IsImported() && m.sysName == "tidb" {
 				iterErr = errors.New(
 					"'imported' is set to 'true' for new instance, this is only used " +
 						"for instances imported from tidb-ansible and make no sense when " +

pkg/cluster/spec/prometheus.go

@@ -372,7 +372,7 @@ func (i *MonitorInstance) initRules(ctx context.Context, e ctxt.Executor, spec *
 		"mkdir -p %[1]s/conf",
 		`find %[1]s/conf -type f -name "*.rules.yml" -delete`,
 		`find %[1]s/bin/prometheus -maxdepth 1 -type f -name "*.rules.yml" -exec cp {} %[1]s/conf/ \;`,
-		`find %[1]s/conf -maxdepth 1 -type f -name "*.rules.yml" -exec sed -i "s/ENV_LABELS_ENV/%[2]s/g" {} \;`,
+		`find %[1]s/conf -maxdepth 1 -type f -name "*.rules.yml" -exec sed -i -e "s/ENV_LABELS_ENV/%[2]s/g" {} \;`,
 	}
 	_, stderr, err := e.Execute(ctx, fmt.Sprintf(strings.Join(cmds, " && "), paths.Deploy, clusterName), false)
 	if err != nil {

pkg/cluster/spec/server_config.go

@@ -143,15 +143,32 @@ func FlattenMap(ms map[string]interface{}) map[string]interface{} {
 	return result
 }
 
-func merge(orig map[string]interface{}, overwrites ...map[string]interface{}) (map[string]interface{}, error) {
+// MergeConfig merge two or more config into one and unflat them
+// config1:
+//   a.b.a: 1
+//   a.b.b: 2
+// config2:
+//   a.b.a: 3
+//   a.b.c: 4
+// config3:
+//   b.c = 5
+// After MergeConfig(config1, config2, config3):
+//   a:
+//     b:
+//       a: 3
+//       b: 2
+//       c: 4
+//   b:
+//     c: 5
+func MergeConfig(orig map[string]interface{}, overwrites ...map[string]interface{}) map[string]interface{} {
 	lhs := FoldMap(orig)
 	for _, overwrite := range overwrites {
 		rhs := FoldMap(overwrite)
 		for k, v := range rhs {
 			patch(lhs, k, v)
 		}
 	}
-	return lhs, nil
+	return lhs
 }
 
 // GetValueFromPath try to find the value by path recursively
@@ -196,11 +213,7 @@ func Merge2Toml(comp string, global, overwrite map[string]interface{}) ([]byte,
 }
 
 func merge2Toml(comp string, global, overwrite map[string]interface{}) ([]byte, error) {
-	lhs, err := merge(global, overwrite)
-	if err != nil {
-		return nil, err
-	}
-
+	lhs := MergeConfig(global, overwrite)
 	buf := bytes.NewBufferString(fmt.Sprintf(`# WARNING: This file is auto-generated. Do not edit! All your modification will be overwritten!
 # You can use 'tiup cluster edit-config' and 'tiup cluster reload' to update the configuration
 # All configuration items you want to change can be added to:
@@ -212,7 +225,7 @@ func merge2Toml(comp string, global, overwrite map[string]interface{}) ([]byte,
 
 	enc := toml.NewEncoder(buf)
 	enc.Indent = ""
-	err = enc.Encode(lhs)
+	err := enc.Encode(lhs)
 	if err != nil {
 		return nil, perrs.Trace(err)
 	}
@@ -240,10 +253,7 @@ func mergeImported(importConfig []byte, specConfigs ...map[string]interface{}) (
 	}
 
 	// overwrite topology specifieced configs upon the imported configs
-	lhs, err := merge(configData, specConfigs...)
-	if err != nil {
-		return nil, perrs.Trace(err)
-	}
+	lhs := MergeConfig(configData, specConfigs...)
 	return lhs, nil
 }
 

pkg/cluster/spec/tiflash.go

@@ -437,11 +437,7 @@ server_configs:
 		return nil, err
 	}
 
-	conf, err := merge(topo.ServerConfigs.TiFlash, src)
-	if err != nil {
-		return nil, err
-	}
-
+	conf := MergeConfig(topo.ServerConfigs.TiFlash, src)
 	return conf, nil
 }
 
@@ -458,10 +454,7 @@ func (i *TiFlashInstance) mergeTiFlashInstanceConfig(clusterVersion string, glob
 		delete(globalConf, "path")
 	}
 
-	conf, err = merge(globalConf, instanceConf)
-	if err != nil {
-		return nil, err
-	}
+	conf = MergeConfig(globalConf, instanceConf)
 	return conf, nil
 }
 
@@ -500,11 +493,7 @@ server_configs:
 		return nil, err
 	}
 
-	conf, err := merge(topo.ServerConfigs.TiFlashLearner, src)
-	if err != nil {
-		return nil, err
-	}
-
+	conf := MergeConfig(topo.ServerConfigs.TiFlashLearner, src)
 	return conf, nil
 }
 

pkg/cluster/spec/validate.go

@@ -223,6 +223,9 @@ func CheckClusterDirOverlap(entries []DirEntry) error {
 			}
 
 			if utils.IsSubDir(d1.dir, d2.dir) || utils.IsSubDir(d2.dir, d1.dir) {
+				if d1.instance.IsImported() && d2.instance.IsImported() {
+					continue
+				}
 				properties := map[string]string{
 					"ThisDirKind":   d1.dirKind,
 					"ThisDir":       d1.dir,

pkg/cluster/template/scripts/pd.go

@@ -66,6 +66,8 @@ func (c *PDScript) WithListenHost(listenHost string) *PDScript {
 // WithScheme set Scheme field of PDScript
 func (c *PDScript) WithScheme(scheme string) *PDScript {
 	c.Scheme = scheme
+	c.AdvertiseClientAddr = fmt.Sprintf("%s://%s:%d", c.Scheme, c.IP, c.ClientPort)
+	c.AdvertisePeerAddr = fmt.Sprintf("%s://%s:%d", c.Scheme, c.IP, c.PeerPort)
 	return c
 }
 

tests/tiup-playground/test_playground.sh

@@ -47,15 +47,16 @@ function kill_all() {
     killall -9 grafana-server || true
     killall -9 tiup-playground || true
     killall -9 prometheus || true
+    cat $outfile
 }
 
 outfile=/tmp/tiup-playground-test.out
-tiup-playground v4.0.10 --tiflash 0 > $outfile 2>&1 &
+tiup-playground nightly > $outfile 2>&1 &
 
 # wait $outfile generated
 sleep 3
 
-trap "kill_all > /dev/null 2>&1" EXIT
+trap "kill_all" EXIT
 
 # wait start cluster successfully
 timeout 300 grep -q "CLUSTER START SUCCESSFULLY" <(tail -f $outfile)
@@ -97,4 +98,17 @@ tiup-playground display | grep -E "terminated|exit" | wc -l | grep -q "1"
 
 killall -2 tiup-playground.test || killall -2 tiup-playground
 
+sleep 60
+
+# test restart with same data
+tiup-playground nightly > $outfile 2>&1 &
+
+# wait $outfile generated
+sleep 3
+
+# wait start cluster successfully
+timeout 300 grep -q "CLUSTER START SUCCESSFULLY" <(tail -f $outfile)
+
+cat $outfile | grep ":3930" | grep -q "Done"
+
 echo -e "\033[0;36m<<< Run all test success >>>\033[0m"