From b4c0a9f59e9ffb1e5e7c20e2515a84fb8844053a Mon Sep 17 00:00:00 2001 From: SIGSEGV Date: Tue, 29 Sep 2020 11:49:01 +0800 Subject: [PATCH] Use absolute path of DataDir/DeployDir/LogDir (#822) --- components/cluster/command/check.go | 6 +- components/dm/spec/logic.go | 17 +- components/dm/spec/topology_dm.go | 6 +- components/dm/spec/topology_dm_test.go | 432 ++++++++++++++---- go.mod | 2 +- go.sum | 8 +- pkg/cluster/clusterutil/clusterutil.go | 40 -- pkg/cluster/clusterutil/parse_topology.go | 65 --- .../clusterutil/parse_topology_test.go | 24 - pkg/cluster/manager.go | 57 +-- pkg/cluster/operation/check.go | 3 +- pkg/cluster/operation/destroy.go | 3 +- pkg/cluster/spec/parse_topology.go | 138 ++++++ pkg/cluster/spec/parse_topology_test.go | 396 ++++++++++++++++ pkg/cluster/spec/spec.go | 9 +- .../testdata/topology_err.yaml | 0 pkg/cluster/spec/util.go | 22 + .../clusterutil_test.go => spec/util_test.go} | 15 +- pkg/cluster/spec/validate.go | 7 +- pkg/cluster/task/check.go | 3 +- 20 files changed, 967 insertions(+), 286 deletions(-) delete mode 100644 pkg/cluster/clusterutil/clusterutil.go delete mode 100644 pkg/cluster/clusterutil/parse_topology.go delete mode 100644 pkg/cluster/clusterutil/parse_topology_test.go create mode 100644 pkg/cluster/spec/parse_topology.go create mode 100644 pkg/cluster/spec/parse_topology_test.go rename pkg/cluster/{clusterutil => spec}/testdata/topology_err.yaml (100%) rename pkg/cluster/{clusterutil/clusterutil_test.go => spec/util_test.go} (53%) diff --git a/components/cluster/command/check.go b/components/cluster/command/check.go index 20898c6958..066fbc20c7 100644 --- a/components/cluster/command/check.go +++ b/components/cluster/command/check.go @@ -23,7 +23,6 @@ import ( "github.com/joomcode/errorx" perrs "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cliutil" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" @@ -82,9 +81,10 @@ conflict checks with other clusters`, topo = *metadata.Topology } else { // check before cluster is deployed - if err := clusterutil.ParseTopologyYaml(args[0], &topo); err != nil { + if err := spec.ParseTopologyYaml(args[0], &topo); err != nil { return err } + spec.ExpandRelativeDir(&topo) clusterList, err := tidbSpec.GetAllClusters() if err != nil { @@ -266,7 +266,7 @@ func checkSystemInfo(s *cliutil.SSHConnectionProps, topo *spec.Specification, op // if the data dir set in topology is relative, and the home dir of deploy user // and the user run the check command is on different partitions, the disk detection // may be using incorrect partition for validations. - for _, dataDir := range clusterutil.MultiDirAbs(opt.user, inst.DataDir()) { + for _, dataDir := range spec.MultiDirAbs(opt.user, inst.DataDir()) { // build checking tasks t2 = t2. CheckSys( diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go index 735e3f95ee..cc564dca3f 100644 --- a/components/dm/spec/logic.go +++ b/components/dm/spec/logic.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/template/scripts" @@ -365,25 +364,25 @@ func (topo *Topology) IterHost(fn func(instance Instance)) { // Endpoints returns the PD endpoints configurations func (topo *Topology) Endpoints(user string) []*scripts.DMMasterScript { var ends []*scripts.DMMasterScript - for _, spec := range topo.Masters { - deployDir := clusterutil.Abs(user, spec.DeployDir) + for _, s := range topo.Masters { + deployDir := spec.Abs(user, s.DeployDir) // data dir would be empty for components which don't need it - dataDir := spec.DataDir + dataDir := s.DataDir // the default data_dir is relative to deploy_dir if dataDir != "" && !strings.HasPrefix(dataDir, "/") { dataDir = filepath.Join(deployDir, dataDir) } // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(user, spec.LogDir) + logDir := spec.Abs(user, s.LogDir) script := scripts.NewDMMasterScript( - spec.Name, - spec.Host, + s.Name, + s.Host, deployDir, dataDir, logDir). - WithPort(spec.Port). - WithPeerPort(spec.PeerPort) + WithPort(s.Port). + WithPeerPort(s.PeerPort) ends = append(ends, script) } return ends diff --git a/components/dm/spec/topology_dm.go b/components/dm/spec/topology_dm.go index 84da44bb51..d0c92312bf 100644 --- a/components/dm/spec/topology_dm.go +++ b/components/dm/spec/topology_dm.go @@ -24,7 +24,6 @@ import ( "github.com/creasty/defaults" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" @@ -479,7 +478,7 @@ func (topo *Topology) CountDir(targetHost, dirPrefix string) int { dirStats := make(map[string]int) count := 0 topoSpec := reflect.ValueOf(topo).Elem() - dirPrefix = clusterutil.Abs(topo.GlobalOptions.User, dirPrefix) + dirPrefix = spec.Abs(topo.GlobalOptions.User, dirPrefix) for i := 0; i < topoSpec.NumField(); i++ { if isSkipField(topoSpec.Field(i)) { @@ -516,7 +515,7 @@ func (topo *Topology) CountDir(targetHost, dirPrefix string) int { dir = filepath.Join(deployDir, dir) } } - dir = clusterutil.Abs(topo.GlobalOptions.User, dir) + dir = spec.Abs(topo.GlobalOptions.User, dir) dirStats[host+dir]++ } } @@ -682,6 +681,7 @@ func setDMCustomDefaults(globalOptions *GlobalOptions, field reflect.Value) erro ))) continue } + // If the data dir in global options is empty or a relative path, keep it be relative // Our run_*.sh start scripts are run inside deploy_path, so the final location // will be deploy_path/global.data_dir diff --git a/components/dm/spec/topology_dm_test.go b/components/dm/spec/topology_dm_test.go index 3a689cd92a..fbfa04c528 100644 --- a/components/dm/spec/topology_dm_test.go +++ b/components/dm/spec/topology_dm_test.go @@ -14,7 +14,14 @@ package spec import ( + "fmt" + "io/ioutil" + "os" + "testing" + . "github.com/pingcap/check" + "github.com/pingcap/tiup/pkg/cluster/spec" + "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" ) @@ -23,31 +30,31 @@ type metaSuiteDM struct { var _ = Suite(&metaSuiteDM{}) -func (s *metaSuiteDM) TestDefaultDataDir(c *C) { +func TestDefaultDataDir(t *testing.T) { // Test with without global DataDir. topo := new(Topology) topo.Masters = append(topo.Masters, MasterSpec{Host: "1.1.1.1", Port: 1111}) topo.Workers = append(topo.Workers, WorkerSpec{Host: "1.1.2.1", Port: 2221}) data, err := yaml.Marshal(topo) - c.Assert(err, IsNil) + assert.Nil(t, err) // Check default value. topo = new(Topology) err = yaml.Unmarshal(data, topo) - c.Assert(err, IsNil) - c.Assert(topo.GlobalOptions.DataDir, Equals, "data") - c.Assert(topo.Masters[0].DataDir, Equals, "data") - c.Assert(topo.Workers[0].DataDir, Equals, "data") + assert.Nil(t, err) + assert.Equal(t, "data", topo.GlobalOptions.DataDir) + assert.Equal(t, "data", topo.Masters[0].DataDir) + assert.Equal(t, "data", topo.Workers[0].DataDir) // Can keep the default value. data, err = yaml.Marshal(topo) - c.Assert(err, IsNil) + assert.Nil(t, err) topo = new(Topology) err = yaml.Unmarshal(data, topo) - c.Assert(err, IsNil) - c.Assert(topo.GlobalOptions.DataDir, Equals, "data") - c.Assert(topo.Masters[0].DataDir, Equals, "data") - c.Assert(topo.Workers[0].DataDir, Equals, "data") + assert.Nil(t, err) + assert.Equal(t, "data", topo.GlobalOptions.DataDir) + assert.Equal(t, "data", topo.Masters[0].DataDir) + assert.Equal(t, "data", topo.Workers[0].DataDir) // Test with global DataDir. topo = new(Topology) @@ -57,140 +64,127 @@ func (s *metaSuiteDM) TestDefaultDataDir(c *C) { topo.Workers = append(topo.Workers, WorkerSpec{Host: "1.1.2.1", Port: 2221}) topo.Workers = append(topo.Workers, WorkerSpec{Host: "1.1.2.2", Port: 2222, DataDir: "/my_data"}) data, err = yaml.Marshal(topo) - c.Assert(err, IsNil) + assert.Nil(t, err) topo = new(Topology) err = yaml.Unmarshal(data, topo) - c.Assert(err, IsNil) - c.Assert(topo.GlobalOptions.DataDir, Equals, "/gloable_data") - c.Assert(topo.Masters[0].DataDir, Equals, "/gloable_data/dm-master-1111") - c.Assert(topo.Masters[1].DataDir, Equals, "/my_data") - c.Assert(topo.Workers[0].DataDir, Equals, "/gloable_data/dm-worker-2221") - c.Assert(topo.Workers[1].DataDir, Equals, "/my_data") + + assert.Nil(t, err) + assert.Equal(t, "/gloable_data", topo.GlobalOptions.DataDir) + assert.Equal(t, "/gloable_data/dm-master-1111", topo.Masters[0].DataDir) + assert.Equal(t, "/my_data", topo.Masters[1].DataDir) + assert.Equal(t, "/gloable_data/dm-worker-2221", topo.Workers[0].DataDir) + assert.Equal(t, "/my_data", topo.Workers[1].DataDir) } -func (s *metaSuiteDM) TestGlobalOptions(c *C) { +func TestGlobalOptions(t *testing.T) { topo := Topology{} err := yaml.Unmarshal([]byte(` global: user: "test1" ssh_port: 220 deploy_dir: "test-deploy" - data_dir: "test-data" -dm-master_servers: + data_dir: "test-data" +master_servers: - host: 172.16.5.138 deploy_dir: "master-deploy" -dm-worker_servers: +worker_servers: - host: 172.16.5.53 data_dir: "worker-data" `), &topo) - c.Assert(err, IsNil) - c.Assert(topo.GlobalOptions.User, Equals, "test1") - c.Assert(topo.GlobalOptions.SSHPort, Equals, 220) - c.Assert(topo.Masters[0].SSHPort, Equals, 220) - c.Assert(topo.Masters[0].DeployDir, Equals, "master-deploy") - - c.Assert(topo.Workers[0].SSHPort, Equals, 220) - c.Assert(topo.Workers[0].DeployDir, Equals, "test-deploy/dm-worker-8262") - c.Assert(topo.Workers[0].DataDir, Equals, "worker-data") + assert.Nil(t, err) + assert.Equal(t, "test1", topo.GlobalOptions.User) + + assert.Equal(t, 220, topo.GlobalOptions.SSHPort) + assert.Equal(t, 220, topo.Masters[0].SSHPort) + assert.Equal(t, "master-deploy", topo.Masters[0].DeployDir) + + assert.Equal(t, 220, topo.Workers[0].SSHPort) + assert.Equal(t, "test-deploy/dm-worker-8262", topo.Workers[0].DeployDir) + assert.Equal(t, "worker-data", topo.Workers[0].DataDir) } -func (s *metaSuiteDM) TestDirectoryConflicts(c *C) { +func TestDirectoryConflicts(t *testing.T) { topo := Topology{} err := yaml.Unmarshal([]byte(` global: user: "test1" ssh_port: 220 deploy_dir: "test-deploy" - data_dir: "test-data" -dm-master_servers: + data_dir: "test-data" +master_servers: - host: 172.16.5.138 deploy_dir: "/test-1" -dm-worker_servers: +worker_servers: - host: 172.16.5.138 data_dir: "/test-1" `), &topo) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "directory conflict for '/test-1' between 'dm-master_servers:172.16.5.138.deploy_dir' and 'dm-worker_servers:172.16.5.138.data_dir'") + assert.NotNil(t, err) + assert.Equal(t, "directory conflict for '/test-1' between 'master_servers:172.16.5.138.deploy_dir' and 'worker_servers:172.16.5.138.data_dir'", err.Error()) err = yaml.Unmarshal([]byte(` global: user: "test1" ssh_port: 220 deploy_dir: "test-deploy" - data_dir: "/test-data" -dm-master_servers: + data_dir: "/test-data" +master_servers: - host: 172.16.5.138 data_dir: "test-1" -dm-worker_servers: +worker_servers: - host: 172.16.5.138 data_dir: "test-1" `), &topo) - c.Assert(err, IsNil) + assert.Nil(t, err) } -func (s *metaSuiteDM) TestPortConflicts(c *C) { +func TestPortConflicts(t *testing.T) { topo := Topology{} err := yaml.Unmarshal([]byte(` global: user: "test1" ssh_port: 220 deploy_dir: "test-deploy" - data_dir: "test-data" -dm-master_servers: + data_dir: "test-data" +master_servers: - host: 172.16.5.138 peer_port: 1234 -dm-worker_servers: +worker_servers: - host: 172.16.5.138 port: 1234 `), &topo) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "port conflict for '1234' between 'dm-master_servers:172.16.5.138.peer_port' and 'dm-worker_servers:172.16.5.138.port'") - - topo = Topology{} - err = yaml.Unmarshal([]byte(` -monitored: - node_exporter_port: 1234 -dm-master_servers: - - host: 172.16.5.138 - port: 1234 -dm-worker_servers: - - host: 172.16.5.138 - status_port: 2345 -`), &topo) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "port conflict for '1234' between 'dm-master_servers:172.16.5.138.port' and 'monitored:172.16.5.138.node_exporter_port'") - + assert.NotNil(t, err) + assert.Equal(t, "port conflict for '1234' between 'master_servers:172.16.5.138.peer_port,omitempty' and 'worker_servers:172.16.5.138.port,omitempty'", err.Error()) } -func (s *metaSuiteDM) TestPlatformConflicts(c *C) { +func TestPlatformConflicts(t *testing.T) { // aarch64 and arm64 are equal topo := Topology{} err := yaml.Unmarshal([]byte(` global: os: "linux" arch: "aarch64" -dm-master_servers: +master_servers: - host: 172.16.5.138 arch: "arm64" -dm-worker_servers: +worker_servers: - host: 172.16.5.138 `), &topo) - c.Assert(err, IsNil) + assert.Nil(t, err) // different arch defined for the same host topo = Topology{} err = yaml.Unmarshal([]byte(` global: os: "linux" -dm-master_servers: +master_servers: - host: 172.16.5.138 arch: "aarch64" -dm-worker_servers: +worker_servers: - host: 172.16.5.138 `), &topo) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "platform mismatch for '172.16.5.138' between 'dm-master_servers:linux/arm64' and 'dm-worker_servers:linux/amd64'") + assert.NotNil(t, err) + assert.Equal(t, "platform mismatch for '172.16.5.138' between 'master_servers:linux/arm64' and 'worker_servers:linux/amd64'", err.Error()) // different os defined for the same host topo = Topology{} @@ -198,18 +192,17 @@ dm-worker_servers: global: os: "linux" arch: "aarch64" -dm-master_servers: +master_servers: - host: 172.16.5.138 os: "darwin" -dm-worker_servers: +worker_servers: - host: 172.16.5.138 `), &topo) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "platform mismatch for '172.16.5.138' between 'dm-master_servers:darwin/arm64' and 'dm-worker_servers:linux/arm64'") - + assert.NotNil(t, err) + assert.Equal(t, "platform mismatch for '172.16.5.138' between 'master_servers:darwin/arm64' and 'worker_servers:linux/arm64'", err.Error()) } -func (s *metaSuiteDM) TestCountDir(c *C) { +func TestCountDir(t *testing.T) { topo := Topology{} err := yaml.Unmarshal([]byte(` @@ -217,42 +210,42 @@ global: user: "test1" ssh_port: 220 deploy_dir: "test-deploy" -dm-master_servers: +master_servers: - host: 172.16.5.138 deploy_dir: "master-deploy" data_dir: "/test-data/data-1" -dm-worker_servers: +worker_servers: - host: 172.16.5.53 data_dir: "test-1" `), &topo) - c.Assert(err, IsNil) + assert.Nil(t, err) cnt := topo.CountDir("172.16.5.53", "test-deploy/dm-worker-8262") - c.Assert(cnt, Equals, 3) + assert.Equal(t, 3, cnt) cnt = topo.CountDir("172.16.5.138", "/test-data/data") - c.Assert(cnt, Equals, 0) // should not match partial path + assert.Equal(t, 0, cnt) // should not match partial path err = yaml.Unmarshal([]byte(` global: user: "test1" ssh_port: 220 deploy_dir: "/test-deploy" -dm-master_servers: +master_servers: - host: 172.16.5.138 deploy_dir: "master-deploy" data_dir: "/test-data/data-1" -dm-worker_servers: +worker_servers: - host: 172.16.5.138 data_dir: "/test-data/data-2" `), &topo) - c.Assert(err, IsNil) + assert.Nil(t, err) cnt = topo.CountDir("172.16.5.138", "/test-deploy/dm-worker-8262") - c.Assert(cnt, Equals, 2) + assert.Equal(t, 2, cnt) cnt = topo.CountDir("172.16.5.138", "") - c.Assert(cnt, Equals, 2) + assert.Equal(t, 2, cnt) cnt = topo.CountDir("172.16.5.138", "test-data") - c.Assert(cnt, Equals, 0) + assert.Equal(t, 0, cnt) cnt = topo.CountDir("172.16.5.138", "/test-data") - c.Assert(cnt, Equals, 2) + assert.Equal(t, 2, cnt) err = yaml.Unmarshal([]byte(` global: @@ -260,21 +253,272 @@ global: ssh_port: 220 deploy_dir: "/test-deploy" data_dir: "/test-data" -dm-master_servers: +master_servers: - host: 172.16.5.138 data_dir: "data-1" -dm-worker_servers: +worker_servers: - host: 172.16.5.138 data_dir: "data-2" - host: 172.16.5.53 `), &topo) - c.Assert(err, IsNil) + assert.Nil(t, err) // if per-instance data_dir is set, the global data_dir is ignored, and if it // is a relative path, it will be under the instance's deploy_dir cnt = topo.CountDir("172.16.5.138", "/test-deploy/dm-worker-8262") - c.Assert(cnt, Equals, 3) + assert.Equal(t, 3, cnt) cnt = topo.CountDir("172.16.5.138", "") - c.Assert(cnt, Equals, 0) + assert.Equal(t, 0, cnt) cnt = topo.CountDir("172.16.5.53", "/test-data") - c.Assert(cnt, Equals, 1) + assert.Equal(t, 1, cnt) +} + +func withTempFile(content string, fn func(string)) { + file, err := ioutil.TempFile("/tmp", "topology-test") + if err != nil { + panic(fmt.Sprintf("create temp file: %s", err)) + } + defer os.Remove(file.Name()) + + _, err = file.WriteString(content) + if err != nil { + panic(fmt.Sprintf("write temp file: %s", err)) + } + file.Close() + + fn(file.Name()) +} + +func with2TempFile(content1, content2 string, fn func(string, string)) { + withTempFile(content1, func(file1 string) { + withTempFile(content2, func(file2 string) { + fn(file1, file2) + }) + }) +} + +func merge4test(base, scale string) (*Topology, error) { + baseTopo := Topology{} + if err := spec.ParseTopologyYaml(base, &baseTopo); err != nil { + return nil, err + } + + scaleTopo := baseTopo.NewPart() + if err := spec.ParseTopologyYaml(scale, scaleTopo); err != nil { + return nil, err + } + + mergedTopo := baseTopo.MergeTopo(scaleTopo) + if err := mergedTopo.Validate(); err != nil { + return nil, err + } + + return mergedTopo.(*Topology), nil +} + +func TestRelativePath(t *testing.T) { + // base test + withTempFile(` +master_servers: + - host: 172.16.5.140 +worker_servers: + - host: 172.16.5.140 +`, func(file string) { + topo := Topology{} + err := spec.ParseTopologyYaml(file, &topo) + assert.Nil(t, err) + spec.ExpandRelativeDir(&topo) + assert.Equal(t, "/home/tidb/deploy/dm-master-8261", topo.Masters[0].DeployDir) + assert.Equal(t, "/home/tidb/deploy/dm-worker-8262", topo.Workers[0].DeployDir) + }) + + // test data dir & log dir + withTempFile(` +master_servers: + - host: 172.16.5.140 + deploy_dir: my-deploy + data_dir: my-data + log_dir: my-log +`, func(file string) { + topo := Topology{} + err := spec.ParseTopologyYaml(file, &topo) + assert.Nil(t, err) + spec.ExpandRelativeDir(&topo) + + assert.Equal(t, "/home/tidb/my-deploy", topo.Masters[0].DeployDir) + assert.Equal(t, "/home/tidb/my-deploy/my-data", topo.Masters[0].DataDir) + assert.Equal(t, "/home/tidb/my-deploy/my-log", topo.Masters[0].LogDir) + }) + + // test global options, case 1 + withTempFile(` +global: + deploy_dir: my-deploy +master_servers: + - host: 172.16.5.140 +`, func(file string) { + topo := Topology{} + err := spec.ParseTopologyYaml(file, &topo) + assert.Nil(t, err) + spec.ExpandRelativeDir(&topo) + + assert.Equal(t, "/home/tidb/my-deploy/dm-master-8261", topo.Masters[0].DeployDir) + assert.Equal(t, "/home/tidb/my-deploy/dm-master-8261/data", topo.Masters[0].DataDir) + assert.Equal(t, "", topo.Masters[0].LogDir) + }) + + // test global options, case 2 + withTempFile(` +global: + deploy_dir: my-deploy +master_servers: + - host: 172.16.5.140 +worker_servers: + - host: 172.16.5.140 + port: 20160 + - host: 172.16.5.140 + port: 20161 +`, func(file string) { + topo := Topology{} + err := spec.ParseTopologyYaml(file, &topo) + assert.Nil(t, err) + spec.ExpandRelativeDir(&topo) + + assert.Equal(t, "my-deploy", topo.GlobalOptions.DeployDir) + assert.Equal(t, "data", topo.GlobalOptions.DataDir) + + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20160", topo.Workers[0].DeployDir) + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20160/data", topo.Workers[0].DataDir) + + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20161", topo.Workers[1].DeployDir) + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20161/data", topo.Workers[1].DataDir) + }) + + // test global options, case 3 + withTempFile(` +global: + deploy_dir: my-deploy +master_servers: + - host: 172.16.5.140 +worker_servers: + - host: 172.16.5.140 + port: 20160 + data_dir: my-data + log_dir: my-log + - host: 172.16.5.140 + port: 20161 +`, func(file string) { + topo := Topology{} + err := spec.ParseTopologyYaml(file, &topo) + assert.Nil(t, err) + spec.ExpandRelativeDir(&topo) + + assert.Equal(t, "my-deploy", topo.GlobalOptions.DeployDir) + assert.Equal(t, "data", topo.GlobalOptions.DataDir) + + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20160", topo.Workers[0].DeployDir) + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20160/my-data", topo.Workers[0].DataDir) + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20160/my-log", topo.Workers[0].LogDir) + + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20161", topo.Workers[1].DeployDir) + assert.Equal(t, "/home/tidb/my-deploy/dm-worker-20161/data", topo.Workers[1].DataDir) + assert.Equal(t, "", topo.Workers[1].LogDir) + }) + + // test global options, case 4 + withTempFile(` +global: + data_dir: my-global-data + log_dir: my-global-log +master_servers: + - host: 172.16.5.140 +worker_servers: + - host: 172.16.5.140 + port: 20160 + data_dir: my-local-data + log_dir: my-local-log + - host: 172.16.5.140 + port: 20161 +`, func(file string) { + topo := Topology{} + err := spec.ParseTopologyYaml(file, &topo) + assert.Nil(t, err) + spec.ExpandRelativeDir(&topo) + + assert.Equal(t, "deploy", topo.GlobalOptions.DeployDir) + assert.Equal(t, "my-global-data", topo.GlobalOptions.DataDir) + assert.Equal(t, "my-global-log", topo.GlobalOptions.LogDir) + + assert.Equal(t, "/home/tidb/deploy/dm-worker-20160", topo.Workers[0].DeployDir) + assert.Equal(t, "/home/tidb/deploy/dm-worker-20160/my-local-data", topo.Workers[0].DataDir) + assert.Equal(t, "/home/tidb/deploy/dm-worker-20160/my-local-log", topo.Workers[0].LogDir) + + assert.Equal(t, "/home/tidb/deploy/dm-worker-20161", topo.Workers[1].DeployDir) + assert.Equal(t, "/home/tidb/deploy/dm-worker-20161/my-global-data", topo.Workers[1].DataDir) + assert.Equal(t, "/home/tidb/deploy/dm-worker-20161/my-global-log", topo.Workers[1].LogDir) + }) +} + +func TestTopologyMerge(t *testing.T) { + // base test + with2TempFile(` +master_servers: + - host: 172.16.5.140 +worker_servers: + - host: 172.16.5.140 +`, ` +worker_servers: + - host: 172.16.5.139 +`, func(base, scale string) { + topo, err := merge4test(base, scale) + assert.Nil(t, err) + spec.ExpandRelativeDir(topo) + + assert.Equal(t, "/home/tidb/deploy/dm-worker-8262", topo.Workers[0].DeployDir) + assert.Equal(t, "/home/tidb/deploy/dm-worker-8262/data", topo.Workers[0].DataDir) + assert.Equal(t, "", topo.Workers[0].LogDir) + + assert.Equal(t, "/home/tidb/deploy/dm-worker-8262", topo.Workers[1].DeployDir) + assert.Equal(t, "/home/tidb/deploy/dm-worker-8262/data", topo.Workers[1].DataDir) + assert.Equal(t, "", topo.Workers[1].LogDir) + }) + + // test global option overwrite + with2TempFile(` +global: + user: test + deploy_dir: /my-global-deploy +master_servers: + - host: 172.16.5.140 +worker_servers: + - host: 172.16.5.140 + log_dir: my-local-log + data_dir: my-local-data + - host: 172.16.5.175 + deploy_dir: flash-deploy + - host: 172.16.5.141 +`, ` +worker_servers: + - host: 172.16.5.139 + deploy_dir: flash-deploy + - host: 172.16.5.134 +`, func(base, scale string) { + topo, err := merge4test(base, scale) + assert.Nil(t, err) + + spec.ExpandRelativeDir(topo) + + assert.Equal(t, "/my-global-deploy/dm-worker-8262", topo.Workers[0].DeployDir) + assert.Equal(t, "/my-global-deploy/dm-worker-8262/my-local-data", topo.Workers[0].DataDir) + assert.Equal(t, "/my-global-deploy/dm-worker-8262/my-local-log", topo.Workers[0].LogDir) + + assert.Equal(t, "/home/test/flash-deploy", topo.Workers[1].DeployDir) + assert.Equal(t, "/home/test/flash-deploy/data", topo.Workers[1].DataDir) + assert.Equal(t, "/home/test/flash-deploy", topo.Workers[3].DeployDir) + assert.Equal(t, "/home/test/flash-deploy/data", topo.Workers[3].DataDir) + + assert.Equal(t, "/my-global-deploy/dm-worker-8262", topo.Workers[2].DeployDir) + assert.Equal(t, "/my-global-deploy/dm-worker-8262/data", topo.Workers[2].DataDir) + assert.Equal(t, "/my-global-deploy/dm-worker-8262", topo.Workers[4].DeployDir) + assert.Equal(t, "/my-global-deploy/dm-worker-8262/data", topo.Workers[4].DataDir) + }) } diff --git a/go.mod b/go.mod index 3a619b1192..9087d13096 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/stretchr/testify v1.5.1 github.com/tikv/pd v1.1.0-beta.0.20200824114021-f8c45ae287fd github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160 - github.com/tj/go-termd v0.0.1 + github.com/tj/go-termd v0.0.2-0.20200115111609-7f6aeb166380 github.com/xo/usql v0.7.8 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.uber.org/atomic v1.6.0 diff --git a/go.sum b/go.sum index aa095ea147..88a657275c 100644 --- a/go.sum +++ b/go.sum @@ -706,9 +706,9 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk= -github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -809,8 +809,8 @@ github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160 h1:NSWpaDaurcAJY7PkL8Xt0 github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/go-css v0.0.0-20191108133013-220a796d1705 h1:+UA89aFRjPMqdccHd9A0HLNCRDXIoElaDoW2C1V3TzA= github.com/tj/go-css v0.0.0-20191108133013-220a796d1705/go.mod h1:e+JPLQ9wyQCgRnPenX2bo7MJoLphBHz5c1WUqaANSeA= -github.com/tj/go-termd v0.0.1 h1:NRrUrpzPj3jVlimGNMdnNOry0vYgvEkMJcJWZkKAeZI= -github.com/tj/go-termd v0.0.1/go.mod h1:qf28T7t3aasdTnAz6ehff7dfebsK+lAKK53duclZ/yQ= +github.com/tj/go-termd v0.0.2-0.20200115111609-7f6aeb166380 h1:ArXHRnjKvyJJnjV5iE3W5czyo3h2awIBwdfUS382T4Y= +github.com/tj/go-termd v0.0.2-0.20200115111609-7f6aeb166380/go.mod h1:7JlPhw1+Bkn5PLz+kqAfzL8ij69OlQ1a4O5bbY98axo= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= diff --git a/pkg/cluster/clusterutil/clusterutil.go b/pkg/cluster/clusterutil/clusterutil.go deleted file mode 100644 index c9cf149d71..0000000000 --- a/pkg/cluster/clusterutil/clusterutil.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package clusterutil - -import ( - "path/filepath" - "strings" -) - -// Abs returns the absolute path -func Abs(user, path string) string { - if !strings.HasPrefix(path, "/") { - return filepath.Join("/home", user, path) - } - return path -} - -// MultiDirAbs returns the absolute path for multi-dir separated by comma -func MultiDirAbs(user, paths string) []string { - var dirs []string - for _, path := range strings.Split(paths, ",") { - path = strings.TrimSpace(path) - if path == "" { - continue - } - dirs = append(dirs, Abs(user, path)) - } - return dirs -} diff --git a/pkg/cluster/clusterutil/parse_topology.go b/pkg/cluster/clusterutil/parse_topology.go deleted file mode 100644 index cc2c51b9e7..0000000000 --- a/pkg/cluster/clusterutil/parse_topology.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package clusterutil - -import ( - "io/ioutil" - - "github.com/joomcode/errorx" - "github.com/pingcap/tiup/pkg/cliutil" - "github.com/pingcap/tiup/pkg/errutil" - "go.uber.org/zap" - "gopkg.in/yaml.v2" -) - -var ( - errNSTopolohy = errorx.NewNamespace("topology") - // ErrTopologyReadFailed is ErrTopologyReadFailed - ErrTopologyReadFailed = errNSTopolohy.NewType("read_failed", errutil.ErrTraitPreCheck) - // ErrTopologyParseFailed is ErrTopologyParseFailed - ErrTopologyParseFailed = errNSTopolohy.NewType("parse_failed", errutil.ErrTraitPreCheck) -) - -// ParseTopologyYaml read yaml content from `file` and unmarshal it to `out` -func ParseTopologyYaml(file string, out interface{}) error { - suggestionProps := map[string]string{ - "File": file, - } - - zap.L().Debug("Parse topology file", zap.String("file", file)) - - yamlFile, err := ioutil.ReadFile(file) - if err != nil { - return ErrTopologyReadFailed. - Wrap(err, "Failed to read topology file %s", file). - WithProperty(cliutil.SuggestionFromTemplate(` -Please check whether your topology file {{ColorKeyword}}{{.File}}{{ColorReset}} exists and try again. - -To generate a sample topology file: - {{ColorCommand}}{{OsArgs0}} template topology > topo.yaml{{ColorReset}} -`, suggestionProps)) - } - - if err = yaml.UnmarshalStrict(yamlFile, out); err != nil { - return ErrTopologyParseFailed. - Wrap(err, "Failed to parse topology file %s", file). - WithProperty(cliutil.SuggestionFromTemplate(` -Please check the syntax of your topology file {{ColorKeyword}}{{.File}}{{ColorReset}} and try again. -`, suggestionProps)) - } - - zap.L().Debug("Parse topology file succeeded", zap.Any("topology", out)) - - return nil -} diff --git a/pkg/cluster/clusterutil/parse_topology_test.go b/pkg/cluster/clusterutil/parse_topology_test.go deleted file mode 100644 index bdc0478799..0000000000 --- a/pkg/cluster/clusterutil/parse_topology_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package clusterutil - -import ( - "path/filepath" - "testing" - - "github.com/pingcap/check" -) - -func TestUtils(t *testing.T) { - check.TestingT(t) -} - -type topoSuite struct{} - -var _ = check.Suite(&topoSuite{}) - -func (s *topoSuite) TestParseTopologyYaml(c *check.C) { - file := filepath.Join("testdata", "topology_err.yaml") - - mp := make(map[string]interface{}) - err := ParseTopologyYaml(file, &mp) - c.Assert(err, check.IsNil) -} diff --git a/pkg/cluster/manager.go b/pkg/cluster/manager.go index 8a7fbd3043..e31cd3521b 100644 --- a/pkg/cluster/manager.go +++ b/pkg/cluster/manager.go @@ -701,11 +701,11 @@ func (m *Manager) Reload(clusterName string, opt operator.Options, skipRestart b } } - deployDir := clusterutil.Abs(base.User, inst.DeployDir()) + deployDir := spec.Abs(base.User, inst.DeployDir()) // data dir would be empty for components which don't need it - dataDirs := clusterutil.MultiDirAbs(base.User, inst.DataDir()) + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(base.User, inst.LogDir()) + logDir := spec.Abs(base.User, inst.LogDir()) // Download and copy the latest component to remote if the cluster is imported from Ansible tb := task.NewBuilder().UserSSH(inst.GetHost(), inst.GetSSHPort(), base.User, opt.SSHTimeout, opt.SSHType, topo.BaseTopo().GlobalOptions.SSHType) @@ -830,11 +830,11 @@ func (m *Manager) Upgrade(clusterName string, clusterVersion string, opt operato downloadCompTasks = append(downloadCompTasks, t) } - deployDir := clusterutil.Abs(base.User, inst.DeployDir()) + deployDir := spec.Abs(base.User, inst.DeployDir()) // data dir would be empty for components which don't need it - dataDirs := clusterutil.MultiDirAbs(base.User, inst.DataDir()) + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(base.User, inst.LogDir()) + logDir := spec.Abs(base.User, inst.LogDir()) // Deploy component tb := task.NewBuilder() @@ -970,7 +970,7 @@ func (m *Manager) Patch(clusterName string, packagePath string, opt operator.Opt var replacePackageTasks []task.Task for _, inst := range insts { - deployDir := clusterutil.Abs(base.User, inst.DeployDir()) + deployDir := spec.Abs(base.User, inst.DeployDir()) tb := task.NewBuilder() tb.BackupComponent(inst.ComponentName(), base.Version, inst.GetHost(), deployDir). InstallPackage(packagePath, inst.GetHost(), deployDir) @@ -1065,10 +1065,12 @@ func (m *Manager) Deploy( metadata := m.specManager.NewMetadata() topo := metadata.GetTopology() - if err := clusterutil.ParseTopologyYaml(topoFile, topo); err != nil { + if err := spec.ParseTopologyYaml(topoFile, topo); err != nil { return err } + spec.ExpandRelativeDir(topo) + base := topo.BaseTopo() if sshType != "" { base.GlobalOptions.SSHType = sshType @@ -1169,7 +1171,7 @@ func (m *Manager) Deploy( if dir == "" { continue } - dirs = append(dirs, clusterutil.Abs(globalOptions.User, dir)) + dirs = append(dirs, spec.Abs(globalOptions.User, dir)) } // the default, relative path of data dir is under deploy dir if strings.HasPrefix(globalOptions.DataDir, "/") { @@ -1204,11 +1206,11 @@ func (m *Manager) Deploy( // Deploy components to remote topo.IterInstance(func(inst spec.Instance) { version := m.bindVersion(inst.ComponentName(), clusterVersion) - deployDir := clusterutil.Abs(globalOptions.User, inst.DeployDir()) + deployDir := spec.Abs(globalOptions.User, inst.DeployDir()) // data dir would be empty for components which don't need it - dataDirs := clusterutil.MultiDirAbs(globalOptions.User, inst.DataDir()) + dataDirs := spec.MultiDirAbs(globalOptions.User, inst.DataDir()) // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(globalOptions.User, inst.LogDir()) + logDir := spec.Abs(globalOptions.User, inst.LogDir()) // Deploy component // prepare deployment server deployDirs := []string{ @@ -1381,11 +1383,11 @@ func (m *Manager) ScaleIn( if deletedNodes.Exist(instance.ID()) { continue } - deployDir := clusterutil.Abs(base.User, instance.DeployDir()) + deployDir := spec.Abs(base.User, instance.DeployDir()) // data dir would be empty for components which don't need it - dataDirs := clusterutil.MultiDirAbs(base.User, instance.DataDir()) + dataDirs := spec.MultiDirAbs(base.User, instance.DataDir()) // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(base.User, instance.LogDir()) + logDir := spec.Abs(base.User, instance.LogDir()) // Download and copy the latest component to remote if the cluster is imported from Ansible tb := task.NewBuilder() @@ -1492,7 +1494,7 @@ func (m *Manager) ScaleOut( // The no tispark master error is ignored, as if the tispark master is removed from the topology // file for some reason (manual edit, for example), it is still possible to scale-out it to make // the whole topology back to normal state. - if err := clusterutil.ParseTopologyYaml(topoFile, newPart); err != nil && + if err := spec.ParseTopologyYaml(topoFile, newPart); err != nil && !errors.Is(perrs.Cause(err), spec.ErrNoTiSparkMaster) { return err } @@ -1506,6 +1508,7 @@ func (m *Manager) ScaleOut( if err := mergedTopo.Validate(); err != nil { return err } + spec.ExpandRelativeDir(mergedTopo) if topo, ok := topo.(*spec.Specification); ok && !opt.NoLabels { // Check if TiKV's label set correctly @@ -1948,7 +1951,7 @@ func buildScaleOutTask( if dirname == "" { continue } - dirs = append(dirs, clusterutil.Abs(globalOptions.User, dirname)) + dirs = append(dirs, spec.Abs(globalOptions.User, dirname)) } } t := task.NewBuilder(). @@ -1977,11 +1980,11 @@ func buildScaleOutTask( // Deploy the new topology and refresh the configuration newPart.IterInstance(func(inst spec.Instance) { version := m.bindVersion(inst.ComponentName(), base.Version) - deployDir := clusterutil.Abs(base.User, inst.DeployDir()) + deployDir := spec.Abs(base.User, inst.DeployDir()) // data dir would be empty for components which don't need it - dataDirs := clusterutil.MultiDirAbs(base.User, inst.DataDir()) + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(base.User, inst.LogDir()) + logDir := spec.Abs(base.User, inst.LogDir()) deployDirs := []string{ deployDir, logDir, @@ -2065,11 +2068,11 @@ func buildScaleOutTask( hasImported := false mergedTopo.IterInstance(func(inst spec.Instance) { - deployDir := clusterutil.Abs(base.User, inst.DeployDir()) + deployDir := spec.Abs(base.User, inst.DeployDir()) // data dir would be empty for components which don't need it - dataDirs := clusterutil.MultiDirAbs(base.User, inst.DataDir()) + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(base.User, inst.LogDir()) + logDir := spec.Abs(base.User, inst.LogDir()) // Download and copy the latest component to remote if the cluster is imported from Ansible tb := task.NewBuilder() @@ -2204,7 +2207,7 @@ func buildMonitoredDeployTask( BuildAsStep(fmt.Sprintf(" - Download %s:%s (%s/%s)", comp, version, info.os, info.arch))) } - deployDir := clusterutil.Abs(globalOptions.User, monitoredOptions.DeployDir) + deployDir := spec.Abs(globalOptions.User, monitoredOptions.DeployDir) // data dir would be empty for components which don't need it dataDir := monitoredOptions.DataDir // the default data_dir is relative to deploy_dir @@ -2212,7 +2215,7 @@ func buildMonitoredDeployTask( dataDir = filepath.Join(deployDir, dataDir) } // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(globalOptions.User, monitoredOptions.LogDir) + logDir := spec.Abs(globalOptions.User, monitoredOptions.LogDir) // Deploy component t := task.NewBuilder(). UserSSH(host, info.ssh, globalOptions.User, sshTimeout, sshType, globalOptions.SSHType). @@ -2268,7 +2271,7 @@ func refreshMonitoredConfigTask( // monitoring agents for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { for host, info := range uniqueHosts { - deployDir := clusterutil.Abs(globalOptions.User, monitoredOptions.DeployDir) + deployDir := spec.Abs(globalOptions.User, monitoredOptions.DeployDir) // data dir would be empty for components which don't need it dataDir := monitoredOptions.DataDir // the default data_dir is relative to deploy_dir @@ -2276,7 +2279,7 @@ func refreshMonitoredConfigTask( dataDir = filepath.Join(deployDir, dataDir) } // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(globalOptions.User, monitoredOptions.LogDir) + logDir := spec.Abs(globalOptions.User, monitoredOptions.LogDir) // Generate configs t := task.NewBuilder(). UserSSH(host, info.ssh, globalOptions.User, sshTimeout, sshType, globalOptions.SSHType). diff --git a/pkg/cluster/operation/check.go b/pkg/cluster/operation/check.go index c1ee8face3..e909c8f6ef 100644 --- a/pkg/cluster/operation/check.go +++ b/pkg/cluster/operation/check.go @@ -23,7 +23,6 @@ import ( "github.com/AstroProfundis/sysinfo" "github.com/pingcap/tidb-insight/collector/insight" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/module" "github.com/pingcap/tiup/pkg/cluster/spec" @@ -499,7 +498,7 @@ func CheckPartitions(opt *CheckOptions, host string, topo *spec.Specification, r if inst.GetHost() != host { return } - for _, dataDir := range clusterutil.MultiDirAbs(topo.GlobalOptions.User, inst.DataDir()) { + for _, dataDir := range spec.MultiDirAbs(topo.GlobalOptions.User, inst.DataDir()) { if dataDir == "" { continue } diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index 4ceef4615d..bc8896290d 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -20,7 +20,6 @@ import ( "strings" "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/module" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/logger/log" @@ -104,7 +103,7 @@ func DeleteGlobalDirs(getter ExecutorGetter, host string, options *spec.GlobalOp if dir == "" { continue } - dir = clusterutil.Abs(options.User, dir) + dir = spec.Abs(options.User, dir) log.Infof("\tClean directory %s on instance %s", dir, host) diff --git a/pkg/cluster/spec/parse_topology.go b/pkg/cluster/spec/parse_topology.go new file mode 100644 index 0000000000..892d8e212f --- /dev/null +++ b/pkg/cluster/spec/parse_topology.go @@ -0,0 +1,138 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spec + +import ( + "io/ioutil" + "path" + "reflect" + "strings" + + "github.com/joomcode/errorx" + "github.com/pingcap/tiup/pkg/cliutil" + "github.com/pingcap/tiup/pkg/errutil" + "go.uber.org/zap" + "gopkg.in/yaml.v2" +) + +var ( + defaultDeployUser = "tidb" + errNSTopolohy = errorx.NewNamespace("topology") + // ErrTopologyReadFailed is ErrTopologyReadFailed + ErrTopologyReadFailed = errNSTopolohy.NewType("read_failed", errutil.ErrTraitPreCheck) + // ErrTopologyParseFailed is ErrTopologyParseFailed + ErrTopologyParseFailed = errNSTopolohy.NewType("parse_failed", errutil.ErrTraitPreCheck) +) + +// ParseTopologyYaml read yaml content from `file` and unmarshal it to `out` +func ParseTopologyYaml(file string, out Topology) error { + suggestionProps := map[string]string{ + "File": file, + } + + zap.L().Debug("Parse topology file", zap.String("file", file)) + + yamlFile, err := ioutil.ReadFile(file) + if err != nil { + return ErrTopologyReadFailed. + Wrap(err, "Failed to read topology file %s", file). + WithProperty(cliutil.SuggestionFromTemplate(` +Please check whether your topology file {{ColorKeyword}}{{.File}}{{ColorReset}} exists and try again. + +To generate a sample topology file: + {{ColorCommand}}{{OsArgs0}} template topology > topo.yaml{{ColorReset}} +`, suggestionProps)) + } + + if err = yaml.UnmarshalStrict(yamlFile, out); err != nil { + return ErrTopologyParseFailed. + Wrap(err, "Failed to parse topology file %s", file). + WithProperty(cliutil.SuggestionFromTemplate(` +Please check the syntax of your topology file {{ColorKeyword}}{{.File}}{{ColorReset}} and try again. +`, suggestionProps)) + } + + zap.L().Debug("Parse topology file succeeded", zap.Any("topology", out)) + + return nil +} + +// ExpandRelativeDir fill DeployDir, DataDir and LogDir to absolute path +func ExpandRelativeDir(topo Topology) { + expandRelativePath(deployUser(topo), topo) +} + +func expandRelativePath(user string, topo interface{}) { + v := reflect.ValueOf(topo).Elem() + + switch v.Kind() { + case reflect.Slice: + for i := 0; i < v.Len(); i++ { + ref := reflect.New(v.Index(i).Type()) + ref.Elem().Set(v.Index(i)) + expandRelativePath(user, ref.Interface()) + v.Index(i).Set(ref.Elem()) + } + case reflect.Struct: + // We should deal with DeployDir first, because DataDir and LogDir depends on it + dirs := []string{"DeployDir", "DataDir", "LogDir"} + for _, dir := range dirs { + f := v.FieldByName(dir) + if !f.IsValid() || f.String() == "" { + continue + } + switch dir { + case "DeployDir": + f.SetString(Abs(user, f.String())) + case "DataDir": + // Some components supports multiple data dirs split by comma + ds := strings.Split(f.String(), ",") + ads := []string{} + for _, d := range ds { + if strings.HasPrefix(d, "/") { + ads = append(ads, d) + } else { + ads = append(ads, path.Join(v.FieldByName("DeployDir").String(), d)) + } + } + f.SetString(strings.Join(ads, ",")) + case "LogDir": + if !strings.HasPrefix(f.String(), "/") { + f.SetString(path.Join(v.FieldByName("DeployDir").String(), f.String())) + } + } + } + // Deal with all fields (expandRelativePath will do nothing on string filed) + for i := 0; i < v.NumField(); i++ { + // We don't deal with GlobalOptions because relative path in GlobalOptions.Data has special meaning + if v.Type().Field(i).Name == "GlobalOptions" { + continue + } + ref := reflect.New(v.Field(i).Type()) + ref.Elem().Set(v.Field(i)) + expandRelativePath(user, ref.Interface()) + v.Field(i).Set(ref.Elem()) + } + case reflect.Ptr: + expandRelativePath(user, v.Interface()) + } +} + +func deployUser(topo Topology) string { + base := topo.BaseTopo() + if base.GlobalOptions == nil || base.GlobalOptions.User == "" { + return defaultDeployUser + } + return base.GlobalOptions.User +} diff --git a/pkg/cluster/spec/parse_topology_test.go b/pkg/cluster/spec/parse_topology_test.go new file mode 100644 index 0000000000..a28e02a2ce --- /dev/null +++ b/pkg/cluster/spec/parse_topology_test.go @@ -0,0 +1,396 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spec + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/pingcap/check" +) + +func TestUtils(t *testing.T) { + check.TestingT(t) +} + +type topoSuite struct{} + +var _ = check.Suite(&topoSuite{}) + +func withTempFile(content string, fn func(string)) { + file, err := ioutil.TempFile("/tmp", "topology-test") + if err != nil { + panic(fmt.Sprintf("create temp file: %s", err)) + } + defer os.Remove(file.Name()) + + _, err = file.WriteString(content) + if err != nil { + panic(fmt.Sprintf("write temp file: %s", err)) + } + file.Close() + + fn(file.Name()) +} + +func with2TempFile(content1, content2 string, fn func(string, string)) { + withTempFile(content1, func(file1 string) { + withTempFile(content2, func(file2 string) { + fn(file1, file2) + }) + }) +} + +func (s *topoSuite) TestParseTopologyYaml(c *check.C) { + file := filepath.Join("testdata", "topology_err.yaml") + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) +} + +func (s *topoSuite) TestRelativePath(c *check.C) { + // test relative path + withTempFile(` +tikv_servers: + - host: 172.16.5.140 + deploy_dir: my-deploy +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + ExpandRelativeDir(&topo) + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/my-deploy") + }) + + // test data dir & log dir + withTempFile(` +tikv_servers: + - host: 172.16.5.140 + deploy_dir: my-deploy + data_dir: my-data + log_dir: my-log +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + ExpandRelativeDir(&topo) + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/my-deploy") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "/home/tidb/my-deploy/my-data") + c.Assert(topo.TiKVServers[0].LogDir, check.Equals, "/home/tidb/my-deploy/my-log") + }) + + // test global options, case 1 + withTempFile(` +global: + deploy_dir: my-deploy + +tikv_servers: + - host: 172.16.5.140 +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + ExpandRelativeDir(&topo) + c.Assert(topo.GlobalOptions.DeployDir, check.Equals, "my-deploy") + c.Assert(topo.GlobalOptions.DataDir, check.Equals, "data") + + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/my-deploy/tikv-20160") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "/home/tidb/my-deploy/tikv-20160/data") + }) + + // test global options, case 2 + withTempFile(` +global: + deploy_dir: my-deploy + +tikv_servers: + - host: 172.16.5.140 + port: 20160 + status_port: 20180 + - host: 172.16.5.140 + port: 20161 + status_port: 20181 +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + ExpandRelativeDir(&topo) + c.Assert(topo.GlobalOptions.DeployDir, check.Equals, "my-deploy") + c.Assert(topo.GlobalOptions.DataDir, check.Equals, "data") + + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/my-deploy/tikv-20160") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "/home/tidb/my-deploy/tikv-20160/data") + + c.Assert(topo.TiKVServers[1].DeployDir, check.Equals, "/home/tidb/my-deploy/tikv-20161") + c.Assert(topo.TiKVServers[1].DataDir, check.Equals, "/home/tidb/my-deploy/tikv-20161/data") + }) + + // test global options, case 3 + withTempFile(` +global: + deploy_dir: my-deploy + +tikv_servers: + - host: 172.16.5.140 + port: 20160 + status_port: 20180 + data_dir: my-data + log_dir: my-log + - host: 172.16.5.140 + port: 20161 + status_port: 20181 +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + + c.Assert(topo.MonitoredOptions.DeployDir, check.Equals, "my-deploy/monitor-9100") + c.Assert(topo.MonitoredOptions.DataDir, check.Equals, "data/monitor-9100") + c.Assert(topo.MonitoredOptions.LogDir, check.Equals, "my-deploy/monitor-9100/log") + + ExpandRelativeDir(&topo) + c.Assert(topo.GlobalOptions.DeployDir, check.Equals, "my-deploy") + c.Assert(topo.GlobalOptions.DataDir, check.Equals, "data") + + c.Assert(topo.MonitoredOptions.DeployDir, check.Equals, "/home/tidb/my-deploy/monitor-9100") + c.Assert(topo.MonitoredOptions.DataDir, check.Equals, "/home/tidb/my-deploy/monitor-9100/data/monitor-9100") + c.Assert(topo.MonitoredOptions.LogDir, check.Equals, "/home/tidb/my-deploy/monitor-9100/my-deploy/monitor-9100/log") + + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/my-deploy/tikv-20160") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "/home/tidb/my-deploy/tikv-20160/my-data") + c.Assert(topo.TiKVServers[0].LogDir, check.Equals, "/home/tidb/my-deploy/tikv-20160/my-log") + + c.Assert(topo.TiKVServers[1].DeployDir, check.Equals, "/home/tidb/my-deploy/tikv-20161") + c.Assert(topo.TiKVServers[1].DataDir, check.Equals, "/home/tidb/my-deploy/tikv-20161/data") + c.Assert(topo.TiKVServers[1].LogDir, check.Equals, "") + }) + + // test global options, case 4 + withTempFile(` +global: + data_dir: my-global-data + log_dir: my-global-log + +tikv_servers: + - host: 172.16.5.140 + port: 20160 + status_port: 20180 + data_dir: my-local-data + log_dir: my-local-log + - host: 172.16.5.140 + port: 20161 + status_port: 20181 +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + ExpandRelativeDir(&topo) + c.Assert(topo.GlobalOptions.DeployDir, check.Equals, "deploy") + c.Assert(topo.GlobalOptions.DataDir, check.Equals, "my-global-data") + c.Assert(topo.GlobalOptions.LogDir, check.Equals, "my-global-log") + + c.Assert(topo.MonitoredOptions.DeployDir, check.Equals, "/home/tidb/deploy/monitor-9100") + c.Assert(topo.MonitoredOptions.DataDir, check.Equals, "/home/tidb/deploy/monitor-9100/my-global-data/monitor-9100") + c.Assert(topo.MonitoredOptions.LogDir, check.Equals, "/home/tidb/deploy/monitor-9100/deploy/monitor-9100/log") + + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/deploy/tikv-20160") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "/home/tidb/deploy/tikv-20160/my-local-data") + c.Assert(topo.TiKVServers[0].LogDir, check.Equals, "/home/tidb/deploy/tikv-20160/my-local-log") + + c.Assert(topo.TiKVServers[1].DeployDir, check.Equals, "/home/tidb/deploy/tikv-20161") + c.Assert(topo.TiKVServers[1].DataDir, check.Equals, "/home/tidb/deploy/tikv-20161/my-global-data") + c.Assert(topo.TiKVServers[1].LogDir, check.Equals, "/home/tidb/deploy/tikv-20161/my-global-log") + }) + + // test multiple dir, case 5 + withTempFile(` +tiflash_servers: + - host: 172.16.5.140 + data_dir: /path/to/my-first-data,my-second-data +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + ExpandRelativeDir(&topo) + + c.Assert(topo.TiFlashServers[0].DeployDir, check.Equals, "/home/tidb/deploy/tiflash-9000") + c.Assert(topo.TiFlashServers[0].DataDir, check.Equals, "/path/to/my-first-data,/home/tidb/deploy/tiflash-9000/my-second-data") + c.Assert(topo.TiFlashServers[0].LogDir, check.Equals, "") + }) + + // test global options, case 6 + withTempFile(` +global: + user: test + data_dir: my-global-data + log_dir: my-global-log + +tikv_servers: + - host: 172.16.5.140 + port: 20160 + status_port: 20180 + deploy_dir: my-local-deploy + data_dir: my-local-data + log_dir: my-local-log + - host: 172.16.5.140 + port: 20161 + status_port: 20181 +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + ExpandRelativeDir(&topo) + c.Assert(topo.GlobalOptions.DeployDir, check.Equals, "deploy") + c.Assert(topo.GlobalOptions.DataDir, check.Equals, "my-global-data") + c.Assert(topo.GlobalOptions.LogDir, check.Equals, "my-global-log") + + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/test/my-local-deploy") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "/home/test/my-local-deploy/my-local-data") + c.Assert(topo.TiKVServers[0].LogDir, check.Equals, "/home/test/my-local-deploy/my-local-log") + + c.Assert(topo.TiKVServers[1].DeployDir, check.Equals, "/home/test/deploy/tikv-20161") + c.Assert(topo.TiKVServers[1].DataDir, check.Equals, "/home/test/deploy/tikv-20161/my-global-data") + c.Assert(topo.TiKVServers[1].LogDir, check.Equals, "/home/test/deploy/tikv-20161/my-global-log") + }) +} + +func merge4test(base, scale string) (*Specification, error) { + baseTopo := Specification{} + if err := ParseTopologyYaml(base, &baseTopo); err != nil { + return nil, err + } + + scaleTopo := baseTopo.NewPart() + if err := ParseTopologyYaml(scale, scaleTopo); err != nil { + return nil, err + } + + mergedTopo := baseTopo.MergeTopo(scaleTopo) + if err := mergedTopo.Validate(); err != nil { + return nil, err + } + + return mergedTopo.(*Specification), nil +} + +func (s *topoSuite) TestTopologyMerge(c *check.C) { + // base test + with2TempFile(` +tiflash_servers: + - host: 172.16.5.140 +`, ` +tiflash_servers: + - host: 172.16.5.139 +`, func(base, scale string) { + topo, err := merge4test(base, scale) + c.Assert(err, check.IsNil) + ExpandRelativeDir(topo) + ExpandRelativeDir(topo) // should be idempotent + + c.Assert(topo.TiFlashServers[0].DeployDir, check.Equals, "/home/tidb/deploy/tiflash-9000") + c.Assert(topo.TiFlashServers[0].DataDir, check.Equals, "/home/tidb/deploy/tiflash-9000/data") + + c.Assert(topo.TiFlashServers[1].DeployDir, check.Equals, "/home/tidb/deploy/tiflash-9000") + c.Assert(topo.TiFlashServers[1].DataDir, check.Equals, "/home/tidb/deploy/tiflash-9000/data") + }) + + // test global option overwrite + with2TempFile(` +global: + user: test + deploy_dir: /my-global-deploy + +tiflash_servers: + - host: 172.16.5.140 + log_dir: my-local-log-tiflash + data_dir: my-local-data-tiflash + - host: 172.16.5.175 + deploy_dir: flash-deploy + - host: 172.16.5.141 +`, ` +tiflash_servers: + - host: 172.16.5.139 + deploy_dir: flash-deploy + - host: 172.16.5.134 +`, func(base, scale string) { + topo, err := merge4test(base, scale) + c.Assert(err, check.IsNil) + + ExpandRelativeDir(topo) + + c.Assert(topo.TiFlashServers[0].DeployDir, check.Equals, "/my-global-deploy/tiflash-9000") + c.Assert(topo.TiFlashServers[0].DataDir, check.Equals, "/my-global-deploy/tiflash-9000/my-local-data-tiflash") + c.Assert(topo.TiFlashServers[0].LogDir, check.Equals, "/my-global-deploy/tiflash-9000/my-local-log-tiflash") + + c.Assert(topo.TiFlashServers[1].DeployDir, check.Equals, "/home/test/flash-deploy") + c.Assert(topo.TiFlashServers[1].DataDir, check.Equals, "/home/test/flash-deploy/data") + c.Assert(topo.TiFlashServers[3].DeployDir, check.Equals, "/home/test/flash-deploy") + c.Assert(topo.TiFlashServers[3].DataDir, check.Equals, "/home/test/flash-deploy/data") + + c.Assert(topo.TiFlashServers[2].DeployDir, check.Equals, "/my-global-deploy/tiflash-9000") + c.Assert(topo.TiFlashServers[2].DataDir, check.Equals, "/my-global-deploy/tiflash-9000/data") + c.Assert(topo.TiFlashServers[4].DeployDir, check.Equals, "/my-global-deploy/tiflash-9000") + c.Assert(topo.TiFlashServers[4].DataDir, check.Equals, "/my-global-deploy/tiflash-9000/data") + }) +} + +func (s *topoSuite) TestFixRelativePath(c *check.C) { + // base test + topo := Specification{ + TiKVServers: []TiKVSpec{ + { + DeployDir: "my-deploy", + }, + }, + } + expandRelativePath("tidb", &topo) + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/my-deploy") + + // test data dir & log dir + topo = Specification{ + TiKVServers: []TiKVSpec{ + { + DeployDir: "my-deploy", + DataDir: "my-data", + LogDir: "my-log", + }, + }, + } + expandRelativePath("tidb", &topo) + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "/home/tidb/my-deploy") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "/home/tidb/my-deploy/my-data") + c.Assert(topo.TiKVServers[0].LogDir, check.Equals, "/home/tidb/my-deploy/my-log") + + // test global options + topo = Specification{ + GlobalOptions: GlobalOptions{ + DeployDir: "my-deploy", + DataDir: "my-data", + LogDir: "my-log", + }, + TiKVServers: []TiKVSpec{ + {}, + }, + } + expandRelativePath("tidb", &topo) + c.Assert(topo.GlobalOptions.DeployDir, check.Equals, "my-deploy") + c.Assert(topo.GlobalOptions.DataDir, check.Equals, "my-data") + c.Assert(topo.GlobalOptions.LogDir, check.Equals, "my-log") + c.Assert(topo.TiKVServers[0].DeployDir, check.Equals, "") + c.Assert(topo.TiKVServers[0].DataDir, check.Equals, "") + c.Assert(topo.TiKVServers[0].LogDir, check.Equals, "") +} diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index b3b6b37208..628cb93de2 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -23,7 +23,6 @@ import ( "github.com/creasty/defaults" "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" @@ -597,7 +596,7 @@ func IterHost(topo Topology, fn func(instance Instance)) { func (s *Specification) Endpoints(user string) []*scripts.PDScript { var ends []*scripts.PDScript for _, spec := range s.PDServers { - deployDir := clusterutil.Abs(user, spec.DeployDir) + deployDir := Abs(user, spec.DeployDir) // data dir would be empty for components which don't need it dataDir := spec.DataDir // the default data_dir is relative to deploy_dir @@ -605,7 +604,7 @@ func (s *Specification) Endpoints(user string) []*scripts.PDScript { dataDir = filepath.Join(deployDir, dataDir) } // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(user, spec.LogDir) + logDir := Abs(user, spec.LogDir) script := scripts.NewPDScript( spec.Name, @@ -629,7 +628,7 @@ func (s *Specification) Endpoints(user string) []*scripts.PDScript { func AlertManagerEndpoints(alertmanager []AlertManagerSpec, user string, enableTLS bool) []*scripts.AlertManagerScript { var ends []*scripts.AlertManagerScript for _, spec := range alertmanager { - deployDir := clusterutil.Abs(user, spec.DeployDir) + deployDir := Abs(user, spec.DeployDir) // data dir would be empty for components which don't need it dataDir := spec.DataDir // the default data_dir is relative to deploy_dir @@ -637,7 +636,7 @@ func AlertManagerEndpoints(alertmanager []AlertManagerSpec, user string, enableT dataDir = filepath.Join(deployDir, dataDir) } // log dir will always be with values, but might not used by the component - logDir := clusterutil.Abs(user, spec.LogDir) + logDir := Abs(user, spec.LogDir) script := scripts.NewAlertManagerScript( spec.Host, diff --git a/pkg/cluster/clusterutil/testdata/topology_err.yaml b/pkg/cluster/spec/testdata/topology_err.yaml similarity index 100% rename from pkg/cluster/clusterutil/testdata/topology_err.yaml rename to pkg/cluster/spec/testdata/topology_err.yaml diff --git a/pkg/cluster/spec/util.go b/pkg/cluster/spec/util.go index 4a0151e991..b092449472 100644 --- a/pkg/cluster/spec/util.go +++ b/pkg/cluster/spec/util.go @@ -18,6 +18,7 @@ import ( "fmt" "path/filepath" "reflect" + "strings" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/utils" @@ -133,3 +134,24 @@ func statusByURL(url string, tlsCfg *tls.Config) string { } return "Up" } + +// Abs returns the absolute path +func Abs(user, path string) string { + if !strings.HasPrefix(path, "/") { + return filepath.Join("/home", user, path) + } + return path +} + +// MultiDirAbs returns the absolute path for multi-dir separated by comma +func MultiDirAbs(user, paths string) []string { + var dirs []string + for _, path := range strings.Split(paths, ",") { + path = strings.TrimSpace(path) + if path == "" { + continue + } + dirs = append(dirs, Abs(user, path)) + } + return dirs +} diff --git a/pkg/cluster/clusterutil/clusterutil_test.go b/pkg/cluster/spec/util_test.go similarity index 53% rename from pkg/cluster/clusterutil/clusterutil_test.go rename to pkg/cluster/spec/util_test.go index 219826f96b..5469861569 100644 --- a/pkg/cluster/clusterutil/clusterutil_test.go +++ b/pkg/cluster/spec/util_test.go @@ -1,4 +1,17 @@ -package clusterutil +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spec import ( "github.com/pingcap/check" diff --git a/pkg/cluster/spec/validate.go b/pkg/cluster/spec/validate.go index f104058e07..bf3b182196 100644 --- a/pkg/cluster/spec/validate.go +++ b/pkg/cluster/spec/validate.go @@ -23,7 +23,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cliutil" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/errutil" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" @@ -43,7 +42,7 @@ var ( func fixDir(topo Topology) func(string) string { return func(dir string) string { if dir != "" { - return clusterutil.Abs(topo.BaseTopo().GlobalOptions.User, dir) + return Abs(topo.BaseTopo().GlobalOptions.User, dir) } return dir } @@ -651,7 +650,7 @@ func (s *Specification) CountDir(targetHost, dirPrefix string) int { dirStats := make(map[string]int) count := 0 topoSpec := reflect.ValueOf(s).Elem() - dirPrefix = clusterutil.Abs(s.GlobalOptions.User, dirPrefix) + dirPrefix = Abs(s.GlobalOptions.User, dirPrefix) for i := 0; i < topoSpec.NumField(); i++ { if isSkipField(topoSpec.Field(i)) { @@ -688,7 +687,7 @@ func (s *Specification) CountDir(targetHost, dirPrefix string) int { dir = filepath.Join(deployDir, dir) } } - dir = clusterutil.Abs(s.GlobalOptions.User, dir) + dir = Abs(s.GlobalOptions.User, dir) dirStats[host+dir]++ } } diff --git a/pkg/cluster/task/check.go b/pkg/cluster/task/check.go index e75dda5491..37b3cef4a6 100644 --- a/pkg/cluster/task/check.go +++ b/pkg/cluster/task/check.go @@ -19,7 +19,6 @@ import ( "strings" "time" - "github.com/pingcap/tiup/pkg/cluster/clusterutil" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" ) @@ -149,7 +148,7 @@ func (c *CheckSys) runFIO(ctx *Context) (outRR []byte, outRW []byte, outLat []by return } - dataDir := clusterutil.Abs(c.topo.GlobalOptions.User, c.dataDir) + dataDir := spec.Abs(c.topo.GlobalOptions.User, c.dataDir) testWd := filepath.Join(dataDir, "tiup-fio-test") fioBin := filepath.Join(CheckToolsPathDir, "bin", "fio")