Merge branch 'master' into playground_monitor

components/cluster/command/edit_config.go

@@ -14,10 +14,12 @@
 package command
 
 import (
+	"github.com/pingcap/tiup/pkg/cluster/manager"
 	"github.com/spf13/cobra"
 )
 
 func newEditConfigCmd() *cobra.Command {
+	opt := manager.EditConfigOptions{}
 	cmd := &cobra.Command{
 		Use:   "edit-config <cluster-name>",
 		Short: "Edit TiDB cluster config.\nWill use editor from environment variable `EDITOR`, default use vi",
@@ -30,9 +32,11 @@ func newEditConfigCmd() *cobra.Command {
 			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
-			return cm.EditConfig(clusterName, skipConfirm)
+			return cm.EditConfig(clusterName, opt, skipConfirm)
 		},
 	}
 
+	cmd.Flags().StringVarP(&opt.NewTopoFile, "topology-file", "", opt.NewTopoFile, "Use provided topology file to substitute the original one instead of editing it.")
+
 	return cmd
 }

components/cluster/command/root.go

@@ -195,6 +195,7 @@ func init() {
 		newAuditCmd(),
 		newImportCmd(),
 		newEditConfigCmd(),
+		newShowConfigCmd(),
 		newReloadCmd(),
 		newPatchCmd(),
 		newRenameCmd(),

components/cluster/command/show_config.go

@@ -0,0 +1,38 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func newShowConfigCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "show-config <cluster-name>",
+		Short: "Show TiDB cluster config",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return cmd.Help()
+			}
+
+			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
+			teleCommand = append(teleCommand, scrubClusterName(clusterName))
+
+			return cm.ShowConfig(clusterName)
+		},
+	}
+
+	return cmd
+}

components/dm/command/edit_config.go

@@ -14,10 +14,12 @@
 package command
 
 import (
+	"github.com/pingcap/tiup/pkg/cluster/manager"
 	"github.com/spf13/cobra"
 )
 
 func newEditConfigCmd() *cobra.Command {
+	opt := manager.EditConfigOptions{}
 	cmd := &cobra.Command{
 		Use:   "edit-config <cluster-name>",
 		Short: "Edit DM cluster config",
@@ -28,9 +30,11 @@ func newEditConfigCmd() *cobra.Command {
 
 			clusterName := args[0]
 
-			return cm.EditConfig(clusterName, skipConfirm)
+			return cm.EditConfig(clusterName, opt, skipConfirm)
 		},
 	}
 
+	cmd.Flags().StringVarP(&opt.NewTopoFile, "topology-file", "", opt.NewTopoFile, "Use provided topology file to substitute the original one instead of editing it.")
+
 	return cmd
 }

pkg/cluster/manager/basic.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/task"
 	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/meta"
+	"github.com/pingcap/tiup/pkg/set"
 	"github.com/pingcap/tiup/pkg/tui"
 )
 
@@ -238,3 +239,26 @@ func (m *Manager) RestartCluster(name string, gOpt operator.Options, skipConfirm
 	log.Infof("Restarted cluster `%s` successfully", name)
 	return nil
 }
+
+// getMonitorHosts  get the instance to ignore list if it marks itself as ignore_exporter
+func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) {
+	// monitor
+	uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch
+	noAgentHosts := set.NewStringSet()
+	topo.IterInstance(func(inst spec.Instance) {
+		// add the instance to ignore list if it marks itself as ignore_exporter
+		if inst.IgnoreMonitorAgent() {
+			noAgentHosts.Insert(inst.GetHost())
+		}
+
+		if _, found := uniqueHosts[inst.GetHost()]; !found {
+			uniqueHosts[inst.GetHost()] = hostInfo{
+				ssh:  inst.GetSSHPort(),
+				os:   inst.OS(),
+				arch: inst.Arch(),
+			}
+		}
+	})
+
+	return uniqueHosts, noAgentHosts
+}

pkg/cluster/manager/cleanup.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/fatih/color"
@@ -37,6 +38,11 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper
 		return err
 	}
 
+	// check locked
+	if err := m.specManager.ScaleOutLockedErr(name); err != nil {
+		return err
+	}
+
 	metadata, err := m.meta(name)
 	if err != nil {
 		return err
@@ -51,51 +57,8 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper
 	}
 
 	// calculate file paths to be deleted before the prompt
-	delFileMap := make(map[string]set.StringSet)
-	for _, com := range topo.ComponentsByStopOrder() {
-		instances := com.Instances()
-		retainDataRoles := set.NewStringSet(cleanOpt.RetainDataRoles...)
-		retainDataNodes := set.NewStringSet(cleanOpt.RetainDataNodes...)
-
-		for _, ins := range instances {
-			// not cleaning files of monitor agents if the instance does not have one
-			switch ins.ComponentName() {
-			case spec.ComponentNodeExporter,
-				spec.ComponentBlackboxExporter:
-				if ins.IgnoreMonitorAgent() {
-					continue
-				}
-			}
-
-			// Some data of instances will be retained
-			dataRetained := retainDataRoles.Exist(ins.ComponentName()) ||
-				retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost())
-
-			if dataRetained {
-				continue
-			}
-
-			dataPaths := set.NewStringSet()
-			logPaths := set.NewStringSet()
-
-			if cleanOpt.CleanupData && len(ins.DataDir()) > 0 {
-				for _, dataDir := range strings.Split(ins.DataDir(), ",") {
-					dataPaths.Insert(path.Join(dataDir, "*"))
-				}
-			}
-
-			if cleanOpt.CleanupLog && len(ins.LogDir()) > 0 {
-				for _, logDir := range strings.Split(ins.LogDir(), ",") {
-					logPaths.Insert(path.Join(logDir, "*.log"))
-				}
-			}
-
-			if delFileMap[ins.GetHost()] == nil {
-				delFileMap[ins.GetHost()] = set.NewStringSet()
-			}
-			delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths)
-		}
-	}
+	delFileMap := getCleanupFiles(topo,
+		cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes)
 
 	if !skipConfirm {
 		target := ""
@@ -155,3 +118,146 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper
 	log.Infof("Cleanup cluster `%s` successfully", name)
 	return nil
 }
+
+// cleanupFiles record the file that needs to be cleaned up
+type cleanupFiles struct {
+	cleanupData     bool     // whether to clean up the data
+	cleanupLog      bool     // whether to clean up the log
+	cleanupTLS      bool     // whether to clean up the tls files
+	retainDataRoles []string // roles that don't clean up
+	retainDataNodes []string // roles that don't clean up
+	delFileMap      map[string]set.StringSet
+}
+
+// getCleanupFiles  get the files that need to be deleted
+func getCleanupFiles(topo spec.Topology,
+	cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet {
+	c := &cleanupFiles{
+		cleanupData:     cleanupData,
+		cleanupLog:      cleanupLog,
+		cleanupTLS:      cleanupTLS,
+		retainDataRoles: retainDataRoles,
+		retainDataNodes: retainDataNodes,
+		delFileMap:      make(map[string]set.StringSet),
+	}
+
+	// calculate file paths to be deleted before the prompt
+	c.instanceCleanupFiles(topo)
+	c.monitorCleanupFiles(topo)
+
+	return c.delFileMap
+}
+
+// instanceCleanupFiles get the files that need to be deleted in the component
+func (c *cleanupFiles) instanceCleanupFiles(topo spec.Topology) {
+	for _, com := range topo.ComponentsByStopOrder() {
+		instances := com.Instances()
+		retainDataRoles := set.NewStringSet(c.retainDataRoles...)
+		retainDataNodes := set.NewStringSet(c.retainDataNodes...)
+
+		for _, ins := range instances {
+			// not cleaning files of monitor agents if the instance does not have one
+			// may not work
+			switch ins.ComponentName() {
+			case spec.ComponentNodeExporter,
+				spec.ComponentBlackboxExporter:
+				if ins.IgnoreMonitorAgent() {
+					continue
+				}
+			}
+
+			// Some data of instances will be retained
+			dataRetained := retainDataRoles.Exist(ins.ComponentName()) ||
+				retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost())
+
+			if dataRetained {
+				continue
+			}
+
+			// prevent duplicate directories
+			dataPaths := set.NewStringSet()
+			logPaths := set.NewStringSet()
+			tlsPath := set.NewStringSet()
+
+			if c.cleanupData && len(ins.DataDir()) > 0 {
+				for _, dataDir := range strings.Split(ins.DataDir(), ",") {
+					dataPaths.Insert(path.Join(dataDir, "*"))
+				}
+			}
+
+			if c.cleanupLog && len(ins.LogDir()) > 0 {
+				for _, logDir := range strings.Split(ins.LogDir(), ",") {
+					logPaths.Insert(path.Join(logDir, "*.log"))
+				}
+			}
+
+			// clean tls data
+			if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled {
+				deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir())
+				tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir)
+				tlsPath.Insert(tlsDir)
+			}
+
+			if c.delFileMap[ins.GetHost()] == nil {
+				c.delFileMap[ins.GetHost()] = set.NewStringSet()
+			}
+			c.delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath)
+		}
+	}
+}
+
+// monitorCleanupFiles get the files that need to be deleted in the mointor
+func (c *cleanupFiles) monitorCleanupFiles(topo spec.Topology) {
+	monitoredOptions := topo.BaseTopo().MonitoredOptions
+	if monitoredOptions == nil {
+		return
+	}
+	user := topo.BaseTopo().GlobalOptions.User
+
+	// get the host with monitor installed
+	uniqueHosts, noAgentHosts := getMonitorHosts(topo)
+	retainDataNodes := set.NewStringSet(c.retainDataNodes...)
+
+	// monitoring agents
+	for host := range uniqueHosts {
+		// determine if host don't need to delete
+		dataRetained := noAgentHosts.Exist(host) || retainDataNodes.Exist(host)
+		if dataRetained {
+			continue
+		}
+
+		deployDir := spec.Abs(user, monitoredOptions.DeployDir)
+
+		// prevent duplicate directories
+		dataPaths := set.NewStringSet()
+		logPaths := set.NewStringSet()
+		tlsPath := set.NewStringSet()
+
+		// data dir would be empty for components which don't need it
+		dataDir := monitoredOptions.DataDir
+		if c.cleanupData && len(dataDir) > 0 {
+			// the default data_dir is relative to deploy_dir
+			if !strings.HasPrefix(dataDir, "/") {
+				dataDir = filepath.Join(deployDir, dataDir)
+			}
+			dataPaths.Insert(path.Join(dataDir, "*"))
+		}
+
+		// log dir will always be with values, but might not used by the component
+		logDir := spec.Abs(user, monitoredOptions.LogDir)
+		if c.cleanupLog && len(logDir) > 0 {
+			logPaths.Insert(path.Join(logDir, "*.log"))
+		}
+
+		// clean tls data
+		if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled {
+			tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir)
+			tlsPath.Insert(tlsDir)
+		}
+
+		if c.delFileMap[host] == nil {
+			c.delFileMap[host] = set.NewStringSet()
+		}
+		c.delFileMap[host].Join(logPaths).Join(dataPaths).Join(tlsPath)
+	}
+}