cluster/import: read and set deploy_dir properly on import

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/pingcap/pd/v4 v4.0.0
 	github.com/pingcap/tidb-insight v0.3.1
 	github.com/r3labs/diff v0.0.0-20200627101315-aecd9dd05dd2
-	github.com/relex/aini v1.1.3
+	github.com/relex/aini v1.2.0
 	github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44
 	github.com/shirou/gopsutil v2.20.3+incompatible
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966

go.sum

@@ -693,8 +693,8 @@ github.com/r3labs/diff v0.0.0-20200627101315-aecd9dd05dd2 h1:786HUIrynbbk5PzUf9R
 github.com/r3labs/diff v0.0.0-20200627101315-aecd9dd05dd2/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig=
 github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/relex/aini v1.1.3 h1:xooajfI84UYB+jWdxKe2w1zXXdi89Fc+L6W4CjHb0wg=
-github.com/relex/aini v1.1.3/go.mod h1:oFQyhvkzwi8GChiLukpBHkV2v142ls2L1MTeOSD2vic=
+github.com/relex/aini v1.2.0 h1:brLADqql7rJrQK2jqcF/uyKlOqbx0jkvuBhF38VD9PQ=
+github.com/relex/aini v1.2.0/go.mod h1:oFQyhvkzwi8GChiLukpBHkV2v142ls2L1MTeOSD2vic=
 github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=

pkg/cluster/ansible/import.go

@@ -73,22 +73,29 @@ func parseInventoryFile(invFile io.Reader) (string, *spec.ClusterMeta, *aini.Inv
 
 	// get global vars
 	if grp, ok := inventory.Groups["all"]; ok && len(grp.Hosts) > 0 {
-		for _, host := range grp.Hosts {
-			if host.Vars["process_supervision"] != "systemd" {
-				return "", nil, inventory, errors.New("only support cluster deployed with systemd")
-			}
-			clsName = host.Vars["cluster_name"]
+		// set global variables
+		clsName = grp.Vars["cluster_name"]
+		clsMeta.User = grp.Vars["ansible_user"]
+		clsMeta.Topology.GlobalOptions.User = clsMeta.User
+		clsMeta.Version = grp.Vars["tidb_version"]
+		clsMeta.Topology.GlobalOptions.DeployDir = grp.Vars["deploy_dir"]
+		// deploy_dir and data_dir of monitored need to be set, otherwise they will be
+		// subdirs of deploy_dir in global options
+		clsMeta.Topology.MonitoredOptions.DeployDir = clsMeta.Topology.GlobalOptions.DeployDir
+		clsMeta.Topology.MonitoredOptions.DataDir = filepath.Join(
+			clsMeta.Topology.MonitoredOptions.DeployDir,
+			"data",
+		)
 
-			clsMeta.User = host.Vars["ansible_user"]
-			clsMeta.Topology.GlobalOptions.User = clsMeta.User
-			clsMeta.Version = host.Vars["tidb_version"]
+		if grp.Vars["process_supervision"] != "systemd" {
+			return "", nil, inventory, errors.New("only support cluster deployed with systemd")
+		}
 
-			if enableBinlog, err := strconv.ParseBool(host.Vars["enable_binlog"]); err == nil && enableBinlog {
-				clsMeta.Topology.ServerConfigs.TiDB["binlog.enable"] = enableBinlog
+		if enableBinlog, err := strconv.ParseBool(grp.Vars["enable_binlog"]); err == nil && enableBinlog {
+			if clsMeta.Topology.ServerConfigs.TiDB == nil {
+				clsMeta.Topology.ServerConfigs.TiDB = make(map[string]interface{})
 			}
-
-			// only read the first host, all global vars should be the same
-			break
+			clsMeta.Topology.ServerConfigs.TiDB["binlog.enable"] = enableBinlog
 		}
 	} else {
 		return "", nil, inventory, errors.New("no available host in the inventory file")

pkg/cluster/ansible/import_test.go

@@ -51,6 +51,20 @@ func (s *ansSuite) TestParseInventoryFile(c *C) {
 
 	expected := []byte(`global:
   user: tiops
+  deploy_dir: /home/tiopsimport/ansible-deploy
+monitored:
+  deploy_dir: /home/tiopsimport/ansible-deploy
+  data_dir: /home/tiopsimport/ansible-deploy/data
+server_configs:
+  tidb:
+    binlog.enable: true
+  tikv: {}
+  pd: {}
+  tiflash: {}
+  tiflash-learner: {}
+  pump: {}
+  drainer: {}
+  cdc: {}
 tidb_servers: []
 tikv_servers: []
 tiflash_servers: []
@@ -59,6 +73,7 @@ monitoring_servers: []
 `)
 
 	topo, err := yaml.Marshal(clsMeta.Topology)
+	fmt.Printf("Got initial topo:\n%s\n", topo)
 	c.Assert(err, IsNil)
 	c.Assert(topo, DeepEquals, expected)
 }
@@ -95,7 +110,7 @@ func (s *ansSuite) TestParseGroupVars(c *C) {
 
 	actual, err := yaml.Marshal(metaFull)
 	c.Assert(err, IsNil)
-	fmt.Printf("Got meta:\n%s\n", actual)
+	fmt.Printf("Got initial meta:\n%s\n", actual)
 
 	c.Assert(metaFull, DeepEquals, expected)
 }

pkg/cluster/ansible/test-data/inventory.ini

@@ -80,7 +80,7 @@ enable_ntpd = True
 set_hostname = True
 
 ## binlog trigger
-enable_binlog = False
+enable_binlog = True
 
 # kafka cluster address for monitoring, example:
 # kafka_addrs = "192.168.0.11:9092,192.168.0.12:9092,192.168.0.13:9092"

pkg/cluster/ansible/test-data/meta.yaml

@@ -4,30 +4,41 @@ topology:
   global:
     user: tiops
     ssh_port: 9999
-    deploy_dir: deploy
+    deploy_dir: /home/tiopsimport/ansible-deploy
     data_dir: data
     os: linux
     arch: amd64
   monitored:
     node_exporter_port: 9101
     blackbox_exporter_port: 9115
-    deploy_dir: deploy/monitor-9101
-    data_dir: data/monitor-9101
+    deploy_dir: /home/tiopsimport/ansible-deploy
+    data_dir: /home/tiopsimport/ansible-deploy/data
+    log_dir: /home/tiopsimport/ansible-deploy/log
+  server_configs:
+    tidb:
+      binlog.enable: true
+    tikv: {}
+    pd: {}
+    tiflash: {}
+    tiflash-learner: {}
+    pump: {}
+    drainer: {}
+    cdc: {}
   tidb_servers:
   - host: 172.16.1.218
     ssh_port: 9999
     imported: true
     port: 4000
     status_port: 3399
-    deploy_dir: deploy/tidb-4000
+    deploy_dir: /home/tiopsimport/ansible-deploy/tidb-4000
     arch: amd64
     os: linux
   - host: 172.16.1.219
     ssh_port: 2222
     imported: true
     port: 3397
     status_port: 10080
-    deploy_dir: deploy/tidb-3397
+    deploy_dir: /home/tiopsimport/ansible-deploy/tidb-3397
     arch: amd64
     os: linux
   tikv_servers:
@@ -36,7 +47,7 @@ topology:
     imported: true
     port: 20160
     status_port: 20180
-    deploy_dir: deploy/tikv-20160
+    deploy_dir: /home/tiopsimport/ansible-deploy/tikv-20160
     data_dir: data/tikv-20160
     arch: amd64
     os: linux
@@ -45,7 +56,7 @@ topology:
     imported: true
     port: 20166
     status_port: 20180
-    deploy_dir: deploy/tikv-20166
+    deploy_dir: /home/tiopsimport/ansible-deploy/tikv-20166
     data_dir: data/tikv-20166
     arch: amd64
     os: linux
@@ -54,7 +65,7 @@ topology:
     imported: true
     port: 20160
     status_port: 20180
-    deploy_dir: deploy/tikv-20160
+    deploy_dir: /home/tiopsimport/ansible-deploy/tikv-20160
     data_dir: data/tikv-20160
     arch: amd64
     os: linux
@@ -68,7 +79,7 @@ topology:
     flash_proxy_port: 20170
     flash_proxy_status_port: 20292
     metrics_port: 8234
-    deploy_dir: deploy/tiflash-9000
+    deploy_dir: /home/tiopsimport/ansible-deploy/tiflash-9000
     data_dir: data/tiflash-9000
     arch: amd64
     os: linux
@@ -81,7 +92,7 @@ topology:
     flash_proxy_port: 20170
     flash_proxy_status_port: 20292
     metrics_port: 8234
-    deploy_dir: deploy/tiflash-9000
+    deploy_dir: /home/tiopsimport/ansible-deploy/tiflash-9000
     data_dir: data/tiflash-9000
     arch: amd64
     os: linux
@@ -92,7 +103,7 @@ topology:
     name: TiDB-PD-218
     client_port: 2379
     peer_port: 2380
-    deploy_dir: deploy/pd-2379
+    deploy_dir: /home/tiopsimport/ansible-deploy/pd-2379
     data_dir: data/pd-2379
     arch: amd64
     os: linux
@@ -102,7 +113,7 @@ topology:
     name: pd-172.16.1.219-2379
     client_port: 2379
     peer_port: 2380
-    deploy_dir: deploy/pd-2379
+    deploy_dir: /home/tiopsimport/ansible-deploy/pd-2379
     data_dir: data/pd-2379
     arch: amd64
     os: linux
@@ -112,7 +123,7 @@ topology:
     name: pd-172.16.1.220-2379
     client_port: 2379
     peer_port: 2380
-    deploy_dir: deploy/pd-2379
+    deploy_dir: /home/tiopsimport/ansible-deploy/pd-2379
     data_dir: data/pd-2379
     arch: amd64
     os: linux
@@ -121,26 +132,24 @@ topology:
     ssh_port: 2222
     imported: true
     port: 8250
-    deploy_dir: deploy/pump-8250
+    deploy_dir: /home/tiopsimport/ansible-deploy/pump-8250
     data_dir: data/pump-8250
-    resource_control: {}
     arch: amd64
     os: linux
   - host: 172.16.1.220
     ssh_port: 9999
     imported: true
     port: 8250
-    deploy_dir: deploy/pump-8250
+    deploy_dir: /home/tiopsimport/ansible-deploy/pump-8250
     data_dir: data/pump-8250
-    resource_control: {}
     arch: amd64
     os: linux
   drainer_servers:
   - host: 172.16.1.221
     ssh_port: 9999
     imported: true
     port: 8249
-    deploy_dir: deploy/drainer-8249
+    deploy_dir: /home/tiopsimport/ansible-deploy/drainer-8249
     data_dir: data/drainer-8249
     arch: amd64
     os: linux
@@ -149,7 +158,7 @@ topology:
     ssh_port: 9999
     imported: true
     port: 9090
-    deploy_dir: deploy/prometheus-9090
+    deploy_dir: /home/tiopsimport/ansible-deploy/prometheus-9090
     data_dir: data/prometheus-9090
     storage_retention: 30d
     arch: amd64
@@ -159,7 +168,7 @@ topology:
     ssh_port: 9999
     imported: true
     port: 3000
-    deploy_dir: deploy/grafana-3000
+    deploy_dir: /home/tiopsimport/ansible-deploy/grafana-3000
     arch: amd64
     os: linux
   alertmanager_servers:
@@ -168,7 +177,7 @@ topology:
     imported: true
     web_port: 9093
     cluster_port: 9094
-    deploy_dir: deploy/alertmanager-9093
+    deploy_dir: /home/tiopsimport/ansible-deploy/alertmanager-9093
     data_dir: data/alertmanager-9093
     arch: amd64
     os: linux