diff --git a/components/cluster/command/edit_config.go b/components/cluster/command/edit_config.go index b7fdca2ed6..7166a5c53b 100644 --- a/components/cluster/command/edit_config.go +++ b/components/cluster/command/edit_config.go @@ -14,10 +14,12 @@ package command import ( + "github.com/pingcap/tiup/pkg/cluster/manager" "github.com/spf13/cobra" ) func newEditConfigCmd() *cobra.Command { + opt := manager.EditConfigOptions{} cmd := &cobra.Command{ Use: "edit-config ", Short: "Edit TiDB cluster config.\nWill use editor from environment variable `EDITOR`, default use vi", @@ -30,9 +32,11 @@ func newEditConfigCmd() *cobra.Command { clusterReport.ID = scrubClusterName(clusterName) teleCommand = append(teleCommand, scrubClusterName(clusterName)) - return cm.EditConfig(clusterName, skipConfirm) + return cm.EditConfig(clusterName, opt, skipConfirm) }, } + cmd.Flags().StringVarP(&opt.NewTopoFile, "topology-file", "", opt.NewTopoFile, "Use provided topology file to substitute the original one instead of editing it.") + return cmd } diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go index e1dfa74524..6d2703e5eb 100644 --- a/components/cluster/command/root.go +++ b/components/cluster/command/root.go @@ -195,6 +195,7 @@ func init() { newAuditCmd(), newImportCmd(), newEditConfigCmd(), + newShowConfigCmd(), newReloadCmd(), newPatchCmd(), newRenameCmd(), diff --git a/components/cluster/command/show_config.go b/components/cluster/command/show_config.go new file mode 100644 index 0000000000..8af796f481 --- /dev/null +++ b/components/cluster/command/show_config.go @@ -0,0 +1,38 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "github.com/spf13/cobra" +) + +func newShowConfigCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "show-config ", + Short: "Show TiDB cluster config", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Help() + } + + clusterName := args[0] + clusterReport.ID = scrubClusterName(clusterName) + teleCommand = append(teleCommand, scrubClusterName(clusterName)) + + return cm.ShowConfig(clusterName) + }, + } + + return cmd +} diff --git a/components/dm/command/edit_config.go b/components/dm/command/edit_config.go index 1d3d3e5d59..05c2704f8b 100644 --- a/components/dm/command/edit_config.go +++ b/components/dm/command/edit_config.go @@ -14,10 +14,12 @@ package command import ( + "github.com/pingcap/tiup/pkg/cluster/manager" "github.com/spf13/cobra" ) func newEditConfigCmd() *cobra.Command { + opt := manager.EditConfigOptions{} cmd := &cobra.Command{ Use: "edit-config ", Short: "Edit DM cluster config", @@ -28,9 +30,11 @@ func newEditConfigCmd() *cobra.Command { clusterName := args[0] - return cm.EditConfig(clusterName, skipConfirm) + return cm.EditConfig(clusterName, opt, skipConfirm) }, } + cmd.Flags().StringVarP(&opt.NewTopoFile, "topology-file", "", opt.NewTopoFile, "Use provided topology file to substitute the original one instead of editing it.") + return cmd } diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index 7ad95da9b2..26caf29b2d 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" ) @@ -238,3 +239,26 @@ func (m *Manager) RestartCluster(name string, gOpt operator.Options, skipConfirm log.Infof("Restarted cluster `%s` successfully", name) return nil } + +// getMonitorHosts get the instance to ignore list if it marks itself as ignore_exporter +func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { + // monitor + uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch + noAgentHosts := set.NewStringSet() + topo.IterInstance(func(inst spec.Instance) { + // add the instance to ignore list if it marks itself as ignore_exporter + if inst.IgnoreMonitorAgent() { + noAgentHosts.Insert(inst.GetHost()) + } + + if _, found := uniqueHosts[inst.GetHost()]; !found { + uniqueHosts[inst.GetHost()] = hostInfo{ + ssh: inst.GetSSHPort(), + os: inst.OS(), + arch: inst.Arch(), + } + } + }) + + return uniqueHosts, noAgentHosts +} diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index 336b91beb0..51365d8d85 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "path" + "path/filepath" "strings" "github.com/fatih/color" @@ -37,6 +38,11 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper return err } + // check locked + if err := m.specManager.ScaleOutLockedErr(name); err != nil { + return err + } + metadata, err := m.meta(name) if err != nil { return err @@ -51,51 +57,8 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper } // calculate file paths to be deleted before the prompt - delFileMap := make(map[string]set.StringSet) - for _, com := range topo.ComponentsByStopOrder() { - instances := com.Instances() - retainDataRoles := set.NewStringSet(cleanOpt.RetainDataRoles...) - retainDataNodes := set.NewStringSet(cleanOpt.RetainDataNodes...) - - for _, ins := range instances { - // not cleaning files of monitor agents if the instance does not have one - switch ins.ComponentName() { - case spec.ComponentNodeExporter, - spec.ComponentBlackboxExporter: - if ins.IgnoreMonitorAgent() { - continue - } - } - - // Some data of instances will be retained - dataRetained := retainDataRoles.Exist(ins.ComponentName()) || - retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) - - if dataRetained { - continue - } - - dataPaths := set.NewStringSet() - logPaths := set.NewStringSet() - - if cleanOpt.CleanupData && len(ins.DataDir()) > 0 { - for _, dataDir := range strings.Split(ins.DataDir(), ",") { - dataPaths.Insert(path.Join(dataDir, "*")) - } - } - - if cleanOpt.CleanupLog && len(ins.LogDir()) > 0 { - for _, logDir := range strings.Split(ins.LogDir(), ",") { - logPaths.Insert(path.Join(logDir, "*.log")) - } - } - - if delFileMap[ins.GetHost()] == nil { - delFileMap[ins.GetHost()] = set.NewStringSet() - } - delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths) - } - } + delFileMap := getCleanupFiles(topo, + cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes) if !skipConfirm { target := "" @@ -155,3 +118,146 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper log.Infof("Cleanup cluster `%s` successfully", name) return nil } + +// cleanupFiles record the file that needs to be cleaned up +type cleanupFiles struct { + cleanupData bool // whether to clean up the data + cleanupLog bool // whether to clean up the log + cleanupTLS bool // whether to clean up the tls files + retainDataRoles []string // roles that don't clean up + retainDataNodes []string // roles that don't clean up + delFileMap map[string]set.StringSet +} + +// getCleanupFiles get the files that need to be deleted +func getCleanupFiles(topo spec.Topology, + cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet { + c := &cleanupFiles{ + cleanupData: cleanupData, + cleanupLog: cleanupLog, + cleanupTLS: cleanupTLS, + retainDataRoles: retainDataRoles, + retainDataNodes: retainDataNodes, + delFileMap: make(map[string]set.StringSet), + } + + // calculate file paths to be deleted before the prompt + c.instanceCleanupFiles(topo) + c.monitorCleanupFiles(topo) + + return c.delFileMap +} + +// instanceCleanupFiles get the files that need to be deleted in the component +func (c *cleanupFiles) instanceCleanupFiles(topo spec.Topology) { + for _, com := range topo.ComponentsByStopOrder() { + instances := com.Instances() + retainDataRoles := set.NewStringSet(c.retainDataRoles...) + retainDataNodes := set.NewStringSet(c.retainDataNodes...) + + for _, ins := range instances { + // not cleaning files of monitor agents if the instance does not have one + // may not work + switch ins.ComponentName() { + case spec.ComponentNodeExporter, + spec.ComponentBlackboxExporter: + if ins.IgnoreMonitorAgent() { + continue + } + } + + // Some data of instances will be retained + dataRetained := retainDataRoles.Exist(ins.ComponentName()) || + retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) + + if dataRetained { + continue + } + + // prevent duplicate directories + dataPaths := set.NewStringSet() + logPaths := set.NewStringSet() + tlsPath := set.NewStringSet() + + if c.cleanupData && len(ins.DataDir()) > 0 { + for _, dataDir := range strings.Split(ins.DataDir(), ",") { + dataPaths.Insert(path.Join(dataDir, "*")) + } + } + + if c.cleanupLog && len(ins.LogDir()) > 0 { + for _, logDir := range strings.Split(ins.LogDir(), ",") { + logPaths.Insert(path.Join(logDir, "*.log")) + } + } + + // clean tls data + if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { + deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir()) + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + tlsPath.Insert(tlsDir) + } + + if c.delFileMap[ins.GetHost()] == nil { + c.delFileMap[ins.GetHost()] = set.NewStringSet() + } + c.delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) + } + } +} + +// monitorCleanupFiles get the files that need to be deleted in the mointor +func (c *cleanupFiles) monitorCleanupFiles(topo spec.Topology) { + monitoredOptions := topo.BaseTopo().MonitoredOptions + if monitoredOptions == nil { + return + } + user := topo.BaseTopo().GlobalOptions.User + + // get the host with monitor installed + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + retainDataNodes := set.NewStringSet(c.retainDataNodes...) + + // monitoring agents + for host := range uniqueHosts { + // determine if host don't need to delete + dataRetained := noAgentHosts.Exist(host) || retainDataNodes.Exist(host) + if dataRetained { + continue + } + + deployDir := spec.Abs(user, monitoredOptions.DeployDir) + + // prevent duplicate directories + dataPaths := set.NewStringSet() + logPaths := set.NewStringSet() + tlsPath := set.NewStringSet() + + // data dir would be empty for components which don't need it + dataDir := monitoredOptions.DataDir + if c.cleanupData && len(dataDir) > 0 { + // the default data_dir is relative to deploy_dir + if !strings.HasPrefix(dataDir, "/") { + dataDir = filepath.Join(deployDir, dataDir) + } + dataPaths.Insert(path.Join(dataDir, "*")) + } + + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(user, monitoredOptions.LogDir) + if c.cleanupLog && len(logDir) > 0 { + logPaths.Insert(path.Join(logDir, "*.log")) + } + + // clean tls data + if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + tlsPath.Insert(tlsDir) + } + + if c.delFileMap[host] == nil { + c.delFileMap[host] = set.NewStringSet() + } + c.delFileMap[host].Join(logPaths).Join(dataPaths).Join(tlsPath) + } +} diff --git a/pkg/cluster/manager/edit_config.go b/pkg/cluster/manager/edit_config.go index 37ab7c260f..32486f0b8e 100644 --- a/pkg/cluster/manager/edit_config.go +++ b/pkg/cluster/manager/edit_config.go @@ -31,8 +31,13 @@ import ( "gopkg.in/yaml.v2" ) +// EditConfigOptions contains the options for config edition. +type EditConfigOptions struct { + NewTopoFile string // path to new topology file to substitute the original one +} + // EditConfig lets the user edit the cluster's config. -func (m *Manager) EditConfig(name string, skipConfirm bool) error { +func (m *Manager) EditConfig(name string, opt EditConfigOptions, skipConfirm bool) error { if err := clusterutil.ValidateClusterNameOrError(name); err != nil { return err } @@ -49,7 +54,7 @@ func (m *Manager) EditConfig(name string, skipConfirm bool) error { return perrs.AddStack(err) } - newTopo, err := m.editTopo(topo, data, skipConfirm) + newTopo, err := m.editTopo(topo, data, opt, skipConfirm) if err != nil { return err } @@ -69,34 +74,40 @@ func (m *Manager) EditConfig(name string, skipConfirm bool) error { return nil } +// If the flag --topology-file is specified, the first 2 steps will be skipped. // 1. Write Topology to a temporary file. // 2. Open file in editor. // 3. Check and update Topology. // 4. Save meta file. -func (m *Manager) editTopo(origTopo spec.Topology, data []byte, skipConfirm bool) (spec.Topology, error) { - file, err := os.CreateTemp(os.TempDir(), "*") - if err != nil { - return nil, perrs.AddStack(err) - } +func (m *Manager) editTopo(origTopo spec.Topology, data []byte, opt EditConfigOptions, skipConfirm bool) (spec.Topology, error) { + var name string + if opt.NewTopoFile == "" { + file, err := os.CreateTemp(os.TempDir(), "*") + if err != nil { + return nil, perrs.AddStack(err) + } - name := file.Name() + name = file.Name() - _, err = io.Copy(file, bytes.NewReader(data)) - if err != nil { - return nil, perrs.AddStack(err) - } + _, err = io.Copy(file, bytes.NewReader(data)) + if err != nil { + return nil, perrs.AddStack(err) + } - err = file.Close() - if err != nil { - return nil, perrs.AddStack(err) - } + err = file.Close() + if err != nil { + return nil, perrs.AddStack(err) + } - err = utils.OpenFileInEditor(name) - if err != nil { - return nil, err + err = utils.OpenFileInEditor(name) + if err != nil { + return nil, err + } + } else { + name = opt.NewTopoFile } - // Now user finish editing the file. + // Now user finish editing the file or user has provided the new topology file newData, err := os.ReadFile(name) if err != nil { return nil, perrs.AddStack(err) @@ -107,8 +118,10 @@ func (m *Manager) editTopo(origTopo spec.Topology, data []byte, skipConfirm bool if err != nil { fmt.Print(color.RedString("New topology could not be saved: ")) log.Infof("Failed to parse topology file: %v", err) - if pass, _ := tui.PromptForConfirmNo("Do you want to continue editing? [Y/n]: "); !pass { - return m.editTopo(origTopo, newData, skipConfirm) + if opt.NewTopoFile == "" { + if pass, _ := tui.PromptForConfirmNo("Do you want to continue editing? [Y/n]: "); !pass { + return m.editTopo(origTopo, newData, opt, skipConfirm) + } } log.Infof("Nothing changed.") return nil, nil @@ -118,8 +131,10 @@ func (m *Manager) editTopo(origTopo spec.Topology, data []byte, skipConfirm bool if err := utils.ValidateSpecDiff(origTopo, newTopo); err != nil { fmt.Print(color.RedString("New topology could not be saved: ")) log.Errorf("%s", err) - if pass, _ := tui.PromptForConfirmNo("Do you want to continue editing? [Y/n]: "); !pass { - return m.editTopo(origTopo, newData, skipConfirm) + if opt.NewTopoFile == "" { + if pass, _ := tui.PromptForConfirmNo("Do you want to continue editing? [Y/n]: "); !pass { + return m.editTopo(origTopo, newData, opt, skipConfirm) + } } log.Infof("Nothing changed.") return nil, nil diff --git a/pkg/cluster/manager/show_config.go b/pkg/cluster/manager/show_config.go new file mode 100644 index 0000000000..6723dea4fc --- /dev/null +++ b/pkg/cluster/manager/show_config.go @@ -0,0 +1,46 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "errors" + "fmt" + + perrs "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/cluster/clusterutil" + "github.com/pingcap/tiup/pkg/meta" + "gopkg.in/yaml.v2" +) + +// ShowConfig shows the cluster's config. +func (m *Manager) ShowConfig(name string) error { + if err := clusterutil.ValidateClusterNameOrError(name); err != nil { + return err + } + + metadata, err := m.meta(name) + if err != nil && !errors.Is(perrs.Cause(err), meta.ErrValidate) { + return err + } + + topo := metadata.GetTopology() + + data, err := yaml.Marshal(topo) + if err != nil { + return perrs.AddStack(err) + } + + fmt.Print(string(data)) + return nil +}