Merge branch 'master' into issue-1640

components/cluster/command/clean.go

@@ -33,6 +33,7 @@ You can retain some nodes and roles data when cleanup the cluster, eg:
     $ tiup cluster clean <cluster-name> --all
     $ tiup cluster clean <cluster-name> --log
     $ tiup cluster clean <cluster-name> --data
+    $ tiup cluster clean <cluster-name> --audit-log
     $ tiup cluster clean <cluster-name> --all --ignore-role prometheus
     $ tiup cluster clean <cluster-name> --all --ignore-node 172.16.13.11:9000
     $ tiup cluster clean <cluster-name> --all --ignore-node 172.16.13.12`,
@@ -50,7 +51,7 @@ You can retain some nodes and roles data when cleanup the cluster, eg:
 				cleanOpt.CleanupLog = true
 			}
 
-			if !(cleanOpt.CleanupData || cleanOpt.CleanupLog) {
+			if !(cleanOpt.CleanupData || cleanOpt.CleanupLog || cleanOpt.CleanupAuditLog) {
 				return cmd.Help()
 			}
 
@@ -62,7 +63,8 @@ You can retain some nodes and roles data when cleanup the cluster, eg:
 	cmd.Flags().StringArrayVar(&cleanOpt.RetainDataRoles, "ignore-role", nil, "Specify the roles whose data will be retained")
 	cmd.Flags().BoolVar(&cleanOpt.CleanupData, "data", false, "Cleanup data")
 	cmd.Flags().BoolVar(&cleanOpt.CleanupLog, "log", false, "Cleanup log")
-	cmd.Flags().BoolVar(&cleanALl, "all", false, "Cleanup both log and data")
+	cmd.Flags().BoolVar(&cleanOpt.CleanupAuditLog, "audit-log", false, "Cleanup TiDB-server audit log")
+	cmd.Flags().BoolVar(&cleanALl, "all", false, "Cleanup both log and data (not include audit log)")
 
 	return cmd
 }

pkg/cluster/manager/cleanup.go

@@ -58,42 +58,16 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper
 
 	// calculate file paths to be deleted before the prompt
 	delFileMap := getCleanupFiles(topo,
-		cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes)
+		cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.CleanupAuditLog, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes)
 
 	if !skipConfirm {
-		target := ""
-		switch {
-		case cleanOpt.CleanupData && cleanOpt.CleanupLog:
-			target = "data and log"
-		case cleanOpt.CleanupData:
-			target = "data"
-		case cleanOpt.CleanupLog:
-			target = "log"
-		}
-
-		// build file list string
-		delFileList := ""
-		for host, fileList := range delFileMap {
-			delFileList += fmt.Sprintf("\n%s:", color.CyanString(host))
-			for _, dfp := range fileList.Slice() {
-				delFileList += fmt.Sprintf("\n %s", dfp)
-			}
-		}
-
-		if err := tui.PromptForConfirmOrAbortError(
-			"This operation will stop %s %s cluster %s and clean its' %s.\nNodes will be ignored: %s\nRoles will be ignored: %s\nFiles to be deleted are: %s\nDo you want to continue? [y/N]:",
-			m.sysName,
-			color.HiYellowString(base.Version),
-			color.HiYellowString(name),
-			target,
-			cleanOpt.RetainDataNodes,
-			cleanOpt.RetainDataRoles,
-			delFileList); err != nil {
+		if err := cleanupConfirm(name, m.sysName, base.Version, cleanOpt, delFileMap); err != nil {
 			return err
 		}
-		log.Infof("Cleanup cluster...")
 	}
 
+	log.Infof("Cleanup cluster...")
+
 	b, err := m.sshTaskBuilder(name, topo, base.User, gOpt)
 	if err != nil {
 		return err
@@ -115,27 +89,76 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper
 		return perrs.Trace(err)
 	}
 
-	log.Infof("Cleanup cluster `%s` successfully", name)
+	log.Infof("Cleanup%s in cluster `%s` successfully", cleanTarget(cleanOpt), name)
 	return nil
 }
 
+// checkConfirm
+func cleanupConfirm(clusterName, sysName, version string, cleanOpt operator.Options, delFileMap map[string]set.StringSet) error {
+	log.Warnf("The clean operation will %s %s %s cluster `%s`",
+		color.HiYellowString("stop"), sysName, version, color.HiYellowString(clusterName))
+	if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil {
+		return err
+	}
+
+	// build file list string
+	delFileList := ""
+	for host, fileList := range delFileMap {
+		// target host has no files to delete
+		if len(fileList) == 0 {
+			continue
+		}
+
+		delFileList += fmt.Sprintf("\n%s:", color.CyanString(host))
+		for _, dfp := range fileList.Slice() {
+			delFileList += fmt.Sprintf("\n %s", dfp)
+		}
+	}
+
+	log.Warnf("Clean the clutser %s's%s.\nNodes will be ignored: %s\nRoles will be ignored: %s\nFiles to be deleted are: %s",
+		color.HiYellowString(clusterName), cleanTarget(cleanOpt), cleanOpt.RetainDataNodes,
+		cleanOpt.RetainDataRoles,
+		delFileList)
+	return tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:")
+}
+
+func cleanTarget(cleanOpt operator.Options) string {
+	target := ""
+
+	if cleanOpt.CleanupData {
+		target += " data"
+	}
+
+	if cleanOpt.CleanupLog {
+		target += (" log")
+	}
+
+	if cleanOpt.CleanupAuditLog {
+		target += (" audit-log")
+	}
+
+	return color.HiYellowString(target)
+}
+
 // cleanupFiles record the file that needs to be cleaned up
 type cleanupFiles struct {
 	cleanupData     bool     // whether to clean up the data
 	cleanupLog      bool     // whether to clean up the log
 	cleanupTLS      bool     // whether to clean up the tls files
+	cleanupAuditLog bool     // whether to clean up the tidb server audit log
 	retainDataRoles []string // roles that don't clean up
 	retainDataNodes []string // roles that don't clean up
 	delFileMap      map[string]set.StringSet
 }
 
 // getCleanupFiles  get the files that need to be deleted
 func getCleanupFiles(topo spec.Topology,
-	cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet {
+	cleanupData, cleanupLog, cleanupTLS, cleanupAuditLog bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet {
 	c := &cleanupFiles{
 		cleanupData:     cleanupData,
 		cleanupLog:      cleanupLog,
 		cleanupTLS:      cleanupTLS,
+		cleanupAuditLog: cleanupAuditLog,
 		retainDataRoles: retainDataRoles,
 		retainDataNodes: retainDataNodes,
 		delFileMap:      make(map[string]set.StringSet),
@@ -187,7 +210,19 @@ func (c *cleanupFiles) instanceCleanupFiles(topo spec.Topology) {
 
 			if c.cleanupLog && len(ins.LogDir()) > 0 {
 				for _, logDir := range strings.Split(ins.LogDir(), ",") {
-					logPaths.Insert(path.Join(logDir, "*.log"))
+					// need to judge the audit log of tidb server
+					if ins.ComponentName() == spec.ComponentTiDB {
+						logPaths.Insert(path.Join(logDir, "tidb?[!audit]*.log"))
+						logPaths.Insert(path.Join(logDir, "tidb.log")) // maybe no need deleted
+					} else {
+						logPaths.Insert(path.Join(logDir, "*.log"))
+					}
+				}
+			}
+
+			if c.cleanupAuditLog && ins.ComponentName() == spec.ComponentTiDB {
+				for _, logDir := range strings.Split(ins.LogDir(), ",") {
+					logPaths.Insert(path.Join(logDir, "tidb-audit*.log"))
 				}
 			}
 

pkg/cluster/operation/operation.go

@@ -47,8 +47,9 @@ type Options struct {
 	SSHProxyTimeout     uint64           // timeout in seconds when connecting the proxy host
 
 	// What type of things should we cleanup in clean command
-	CleanupData bool // should we cleanup data
-	CleanupLog  bool // should we clenaup log
+	CleanupData     bool // should we cleanup data
+	CleanupLog      bool // should we clenaup log
+	CleanupAuditLog bool // should we clenaup tidb server auit log
 
 	// Some data will be retained when destroying instances
 	RetainDataRoles []string

pkg/cluster/task/step.go

@@ -96,7 +96,8 @@ func (s *StepDisplay) Execute(ctx context.Context) error {
 	}
 
 	switch s.DisplayMode {
-	case log.DisplayModeJSON:
+	case log.DisplayModeJSON,
+		log.DisplayModePlain:
 		break
 	default:
 		if singleBar, ok := s.progressBar.(*progress.SingleBar); ok {
@@ -126,7 +127,9 @@ func (s *StepDisplay) Execute(ctx context.Context) error {
 
 	switch s.DisplayMode {
 	case log.DisplayModeJSON:
-		_ = printDp(dp)
+		_ = printDpJSON(dp)
+	case log.DisplayModePlain:
+		printDpPlain(dp)
 	default:
 		s.progressBar.UpdateDisplay(dp)
 	}
@@ -153,7 +156,9 @@ func (s *StepDisplay) handleTaskBegin(task Task) {
 	}
 	switch s.DisplayMode {
 	case log.DisplayModeJSON:
-		_ = printDp(dp)
+		_ = printDpJSON(dp)
+	case log.DisplayModePlain:
+		printDpPlain(dp)
 	default:
 		s.progressBar.UpdateDisplay(dp)
 	}
@@ -169,7 +174,9 @@ func (s *StepDisplay) handleTaskProgress(task Task, p string) {
 	}
 	switch s.DisplayMode {
 	case log.DisplayModeJSON:
-		_ = printDp(dp)
+		_ = printDpJSON(dp)
+	case log.DisplayModePlain:
+		printDpPlain(dp)
 	default:
 		s.progressBar.UpdateDisplay(dp)
 	}
@@ -209,7 +216,8 @@ func (ps *ParallelStepDisplay) SetDisplayMode(m log.DisplayMode) *ParallelStepDi
 // Execute implements the Task interface
 func (ps *ParallelStepDisplay) Execute(ctx context.Context) error {
 	switch ps.DisplayMode {
-	case log.DisplayModeJSON:
+	case log.DisplayModeJSON,
+		log.DisplayModePlain:
 		break
 	default:
 		ps.progressBar.StartRenderLoop()
@@ -229,11 +237,20 @@ func (ps *ParallelStepDisplay) String() string {
 	return ps.inner.String()
 }
 
-func printDp(dp *progress.DisplayProps) error {
+func printDpJSON(dp *progress.DisplayProps) error {
 	output, err := json.Marshal(dp)
 	if err != nil {
 		return err
 	}
 	fmt.Println(string(output))
 	return nil
 }
+
+func printDpPlain(dp *progress.DisplayProps) {
+	switch dp.Mode {
+	case progress.ModeError:
+		log.Errorf("progress: %s", dp)
+	default:
+		log.Infof("progress: %s", dp)
+	}
+}

pkg/logger/log/log.go

@@ -24,6 +24,9 @@ import (
 
 var (
 	outputFmt DisplayMode = DisplayModeDefault // global output format of logger
+
+	stdout io.Writer = os.Stdout
+	stderr io.Writer = os.Stderr
 )
 
 // DisplayMode control the output format
@@ -76,19 +79,29 @@ func Debugf(format string, args ...interface{}) {
 // Deprecated: Use zap.L().Info() instead
 func Infof(format string, args ...interface{}) {
 	zap.L().Info(fmt.Sprintf(format, args...))
-	printLog(os.Stdout, "info", format, args...)
+	printLog(stdout, "info", format, args...)
 }
 
 // Warnf output the warning message to console
 // Deprecated: Use zap.L().Warn() instead
 func Warnf(format string, args ...interface{}) {
 	zap.L().Warn(fmt.Sprintf(format, args...))
-	printLog(os.Stderr, "warn", format, args...)
+	printLog(stderr, "warn", format, args...)
 }
 
 // Errorf output the error message to console
 // Deprecated: Use zap.L().Error() instead
 func Errorf(format string, args ...interface{}) {
 	zap.L().Error(fmt.Sprintf(format, args...))
-	printLog(os.Stderr, "error", format, args...)
+	printLog(stderr, "error", format, args...)
+}
+
+// SetStdout redirect stdout to a custom writer
+func SetStdout(w io.Writer) {
+	stdout = w
+}
+
+// SetStderr redirect stderr to a custom writer
+func SetStderr(w io.Writer) {
+	stderr = w
 }

pkg/logger/log/verbose.go

@@ -31,5 +31,5 @@ func Verbose(format string, args ...interface{}) {
 	if !verbose {
 		return
 	}
-	fmt.Fprintln(os.Stderr, "Verbose:", fmt.Sprintf(format, args...))
+	fmt.Fprintln(stderr, "Verbose:", fmt.Sprintf(format, args...))
 }

pkg/tui/progress/display_props.go

@@ -15,6 +15,7 @@ package progress
 
 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 )
 
@@ -72,9 +73,37 @@ func (m *Mode) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
+// String implements string
+func (m Mode) String() string {
+	var s string
+	switch m {
+	case ModeSpinner:
+		s = "spinner"
+	case ModeProgress:
+		s = "progress"
+	case ModeDone:
+		s = "done"
+	case ModeError:
+		s = "error"
+	default:
+		s = "unknown"
+	}
+	return s
+}
+
 // DisplayProps controls the display of the progress bar.
 type DisplayProps struct {
 	Prefix string `json:"prefix,omitempty"`
 	Suffix string `json:"suffix,omitempty"` // If `Mode == Done / Error`, Suffix is not printed
 	Mode   Mode   `json:"mode,omitempty"`
 }
+
+// String implements string
+func (dp *DisplayProps) String() string {
+	return fmt.Sprintf(
+		"(%s) %s: %s",
+		dp.Mode,
+		dp.Prefix,
+		dp.Suffix,
+	)
+}