cluster: implement pure JSON output mode (experimental) (#1617)

components/cluster/command/deploy.go

@@ -79,8 +79,8 @@ func newDeploy() *cobra.Command {
 	return cmd
 }
 
-func postDeployHook(builder *task.Builder, topo spec.Topology) {
-	nodeInfoTask := task.NewBuilder().Func("Check status", func(ctx context.Context) error {
+func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Options) {
+	nodeInfoTask := task.NewBuilder(gOpt.DisplayMode).Func("Check status", func(ctx context.Context) error {
 		var err error
 		teleNodeInfos, err = operator.GetNodeInfo(ctx, topo)
 		_ = err
@@ -92,7 +92,7 @@ func postDeployHook(builder *task.Builder, topo spec.Topology) {
 		builder.ParallelStep("+ Check status", false, nodeInfoTask)
 	}
 
-	enableTask := task.NewBuilder().Func("Setting service auto start on boot", func(ctx context.Context) error {
+	enableTask := task.NewBuilder(gOpt.DisplayMode).Func("Setting service auto start on boot", func(ctx context.Context) error {
 		return operator.Enable(ctx, topo, operator.Options{}, true)
 	}).BuildAsStep("Enable service").SetHidden(true)
 

components/cluster/command/display.go

@@ -88,7 +88,6 @@ func newDisplayCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&showDashboardOnly, "dashboard", false, "Only display TiDB Dashboard information")
 	cmd.Flags().BoolVar(&showVersionOnly, "version", false, "Only display TiDB cluster version")
 	cmd.Flags().BoolVar(&showTiKVLabels, "labels", false, "Only display labels of specified TiKV role or nodes")
-	cmd.Flags().BoolVar(&gOpt.JSON, "json", false, "Output in JSON format when applicable")
 
 	return cmd
 }

components/cluster/command/root.go

@@ -100,6 +100,15 @@ func init() {
 		SilenceErrors: true,
 		Version:       version.NewTiUPVersion().String(),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			switch strings.ToLower(gOpt.DisplayMode) {
+			case "json":
+				log.SetDisplayMode(log.DisplayModeJSON)
+			case "plain", "text":
+				log.SetDisplayMode(log.DisplayModePlain)
+			default:
+				log.SetDisplayMode(log.DisplayModeDefault)
+			}
+
 			var err error
 			var env *tiupmeta.Environment
 			if err = spec.Initialize("cluster"); err != nil {
@@ -124,9 +133,11 @@ func init() {
 
 			if gOpt.NativeSSH {
 				gOpt.SSHType = executor.SSHTypeSystem
-				zap.L().Info("System ssh client will be used",
-					zap.String(localdata.EnvNameNativeSSHClient, os.Getenv(localdata.EnvNameNativeSSHClient)))
-				fmt.Println("The --native-ssh flag has been deprecated, please use --ssh=system")
+				log.Infof(
+					"System ssh client will be used (%s=%s)",
+					localdata.EnvNameNativeSSHClient,
+					os.Getenv(localdata.EnvNameNativeSSHClient))
+				log.Infof("The --native-ssh flag has been deprecated, please use --ssh=system")
 			}
 
 			err = proxy.MaybeStartProxy(gOpt.SSHProxyHost, gOpt.SSHProxyPort, gOpt.SSHProxyUser, gOpt.SSHProxyUsePassword, gOpt.SSHProxyIdentity)
@@ -152,6 +163,7 @@ func init() {
 	rootCmd.PersistentFlags().BoolVar(&gOpt.NativeSSH, "native-ssh", gOpt.NativeSSH, "(EXPERIMENTAL) Use the native SSH client installed on local system instead of the build-in one.")
 	rootCmd.PersistentFlags().StringVar((*string)(&gOpt.SSHType), "ssh", "", "(EXPERIMENTAL) The executor type: 'builtin', 'system', 'none'.")
 	rootCmd.PersistentFlags().IntVarP(&gOpt.Concurrency, "concurrency", "c", 5, "max number of parallel tasks allowed")
+	rootCmd.PersistentFlags().StringVar(&gOpt.DisplayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]")
 	rootCmd.PersistentFlags().StringVar(&gOpt.SSHProxyHost, "ssh-proxy-host", "", "The SSH proxy host used to connect to remote host.")
 	rootCmd.PersistentFlags().StringVar(&gOpt.SSHProxyUser, "ssh-proxy-user", utils.CurrentUser(), "The user name used to login the proxy host.")
 	rootCmd.PersistentFlags().IntVar(&gOpt.SSHProxyPort, "ssh-proxy-port", 22, "The port used to login the proxy host.")
@@ -324,7 +336,7 @@ func Execute() {
 				if err != nil {
 					log.Infof("report failed: %v", err)
 				}
-				fmt.Fprintf(os.Stderr, "report: %s\n", teleReport.String())
+				log.Errorf("report: %s\n", teleReport.String())
 				if data, err := json.Marshal(teleReport); err == nil {
 					log.Debugf("report: %s\n", string(data))
 				}
@@ -335,24 +347,41 @@ func Execute() {
 		f()
 	}
 
-	if err != nil {
-		if errx := errorx.Cast(err); errx != nil {
-			printErrorMessageForErrorX(errx)
-		} else {
-			printErrorMessageForNormalError(err)
+	switch log.GetDisplayMode() {
+	case log.DisplayModeJSON:
+		obj := struct {
+			Code int    `json:"exit_code"`
+			Err  string `json:"error,omitempty"`
+		}{
+			Code: code,
 		}
-
-		if !errorx.HasTrait(err, utils.ErrTraitPreCheck) {
-			logger.OutputDebugLog("tiup-cluster")
+		if err != nil {
+			obj.Err = err.Error()
+		}
+		data, err := json.Marshal(obj)
+		if err != nil {
+			fmt.Printf("{\"exit_code\":%d, \"error\":\"%s\"}", code, err)
 		}
+		fmt.Fprintln(os.Stderr, string(data))
+	default:
+		if err != nil {
+			if errx := errorx.Cast(err); errx != nil {
+				printErrorMessageForErrorX(errx)
+			} else {
+				printErrorMessageForNormalError(err)
+			}
+
+			if !errorx.HasTrait(err, utils.ErrTraitPreCheck) {
+				logger.OutputDebugLog("tiup-cluster")
+			}
 
-		if errx := errorx.Cast(err); errx != nil {
-			if suggestion := extractSuggestionFromErrorX(errx); len(suggestion) > 0 {
-				_, _ = fmt.Fprintf(os.Stderr, "\n%s\n", suggestion)
+			if errx := errorx.Cast(err); errx != nil {
+				if suggestion := extractSuggestionFromErrorX(errx); len(suggestion) > 0 {
+					log.Errorf("\n%s\n", suggestion)
+				}
 			}
 		}
 	}
-
 	err = logger.OutputAuditLogIfEnabled()
 	if err != nil {
 		zap.L().Warn("Write audit log file failed", zap.Error(err))

components/cluster/command/scale_out.go

@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 
 	"github.com/pingcap/tiup/pkg/cluster/manager"
+	operator "github.com/pingcap/tiup/pkg/cluster/operation"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/cluster/task"
 	"github.com/pingcap/tiup/pkg/utils"
@@ -67,14 +68,14 @@ func newScaleOutCmd() *cobra.Command {
 	return cmd
 }
 
-func final(builder *task.Builder, name string, meta spec.Metadata) {
+func final(builder *task.Builder, name string, meta spec.Metadata, gOpt operator.Options) {
 	builder.UpdateTopology(name,
 		tidbSpec.Path(name),
 		meta.(*spec.ClusterMeta),
 		nil, /* deleteNodeIds */
 	)
 }
 
-func postScaleOutHook(builder *task.Builder, newPart spec.Topology) {
-	postDeployHook(builder, newPart)
+func postScaleOutHook(builder *task.Builder, newPart spec.Topology, gOpt operator.Options) {
+	postDeployHook(builder, newPart, gOpt)
 }

components/dm/command/deploy.go

@@ -81,8 +81,8 @@ func supportVersion(vs string) error {
 	return nil
 }
 
-func postDeployHook(builder *task.Builder, topo spec.Topology) {
-	enableTask := task.NewBuilder().Func("Setting service auto start on boot", func(ctx context.Context) error {
+func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Options) {
+	enableTask := task.NewBuilder(gOpt.DisplayMode).Func("Setting service auto start on boot", func(ctx context.Context) error {
 		return operator.Enable(ctx, topo, operator.Options{}, true)
 	}).BuildAsStep("Enable service").SetHidden(true)
 

components/dm/command/root.go

@@ -14,6 +14,7 @@
 package command
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -119,6 +120,7 @@ please backup your data before process.`,
 	rootCmd.PersistentFlags().BoolVar(&gOpt.NativeSSH, "native-ssh", gOpt.NativeSSH, "Use the SSH client installed on local system instead of the build-in one.")
 	rootCmd.PersistentFlags().StringVar((*string)(&gOpt.SSHType), "ssh", "", "The executor type: 'builtin', 'system', 'none'")
 	rootCmd.PersistentFlags().IntVarP(&gOpt.Concurrency, "concurrency", "c", 5, "max number of parallel tasks allowed")
+	rootCmd.PersistentFlags().StringVar(&gOpt.DisplayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]")
 	rootCmd.PersistentFlags().StringVar(&gOpt.SSHProxyHost, "ssh-proxy-host", "", "The SSH proxy host used to connect to remote host.")
 	rootCmd.PersistentFlags().StringVar(&gOpt.SSHProxyUser, "ssh-proxy-user", utils.CurrentUser(), "The user name used to login the proxy host.")
 	rootCmd.PersistentFlags().IntVar(&gOpt.SSHProxyPort, "ssh-proxy-port", 22, "The port used to login the proxy host.")
@@ -227,19 +229,34 @@ func Execute() {
 	zap.L().Info("Execute command finished", zap.Int("code", code), zap.Error(err))
 
 	if err != nil {
-		if errx := errorx.Cast(err); errx != nil {
-			printErrorMessageForErrorX(errx)
-		} else {
-			printErrorMessageForNormalError(err)
-		}
+		switch strings.ToLower(gOpt.DisplayMode) {
+		case "json":
+			obj := struct {
+				Err string `json:"error"`
+			}{
+				Err: err.Error(),
+			}
+			data, err := json.Marshal(obj)
+			if err != nil {
+				fmt.Printf("{\"error\": \"%s\"}", err)
+				break
+			}
+			fmt.Fprintln(os.Stderr, string(data))
+		default:
+			if errx := errorx.Cast(err); errx != nil {
+				printErrorMessageForErrorX(errx)
+			} else {
+				printErrorMessageForNormalError(err)
+			}
 
-		if !errorx.HasTrait(err, utils.ErrTraitPreCheck) {
-			logger.OutputDebugLog("tiup-dm")
-		}
+			if !errorx.HasTrait(err, utils.ErrTraitPreCheck) {
+				logger.OutputDebugLog("tiup-dm")
+			}
 
-		if errx := errorx.Cast(err); errx != nil {
-			if suggestion := extractSuggestionFromErrorX(errx); len(suggestion) > 0 {
-				_, _ = fmt.Fprintf(os.Stderr, "\n%s\n", suggestion)
+			if errx := errorx.Cast(err); errx != nil {
+				if suggestion := extractSuggestionFromErrorX(errx); len(suggestion) > 0 {
+					_, _ = fmt.Fprintf(os.Stderr, "\n%s\n", suggestion)
+				}
 			}
 		}
 	}

components/dm/command/scale_out.go

@@ -17,6 +17,7 @@ import (
 	"path/filepath"
 
 	"github.com/pingcap/tiup/pkg/cluster/manager"
+	operator "github.com/pingcap/tiup/pkg/cluster/operation"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/cluster/task"
 	"github.com/pingcap/tiup/pkg/utils"
@@ -50,6 +51,6 @@ func newScaleOutCmd() *cobra.Command {
 	return cmd
 }
 
-func postScaleOutHook(builder *task.Builder, newPart spec.Topology) {
-	postDeployHook(builder, newPart)
+func postScaleOutHook(builder *task.Builder, newPart spec.Topology, gOpt operator.Options) {
+	postDeployHook(builder, newPart, gOpt)
 }

pkg/cluster/ansible/config.go

@@ -51,7 +51,7 @@ func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options)
 		for _, inst := range comp.Instances() {
 			switch inst.ComponentName() {
 			case spec.ComponentPD, spec.ComponentTiKV, spec.ComponentPump, spec.ComponentTiDB, spec.ComponentDrainer:
-				t := task.NewBuilder().
+				t := task.NewBuilder("").
 					SSHKeySet(
 						spec.ClusterPath(name, "ssh", "id_rsa"),
 						spec.ClusterPath(name, "ssh", "id_rsa.pub")).
@@ -84,7 +84,7 @@ func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options)
 					Build()
 				copyFileTasks = append(copyFileTasks, t)
 			case spec.ComponentTiFlash:
-				t := task.NewBuilder().
+				t := task.NewBuilder("").
 					SSHKeySet(
 						spec.ClusterPath(name, "ssh", "id_rsa"),
 						spec.ClusterPath(name, "ssh", "id_rsa.pub")).
@@ -131,7 +131,7 @@ func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options)
 			}
 		}
 	}
-	t := task.NewBuilder().
+	t := task.NewBuilder("").
 		Parallel(false, copyFileTasks...).
 		Build()
 

pkg/cluster/audit/audit.go

@@ -31,6 +31,11 @@ import (
 	tiuputils "github.com/pingcap/tiup/pkg/utils"
 )
 
+const (
+	// EnvNameAuditID is the alternative ID appended to time based audit ID
+	EnvNameAuditID = "TIUP_AUDIT_ID"
+)
+
 // CommandArgs returns the original commands from the first line of a file
 func CommandArgs(fp string) ([]string, error) {
 	file, err := os.Open(fp)
@@ -140,7 +145,11 @@ func GetAuditList(dir string) ([]Item, error) {
 
 // OutputAuditLog outputs audit log.
 func OutputAuditLog(dir string, data []byte) error {
-	fname := filepath.Join(dir, base52.Encode(time.Now().UnixNano()+rand.Int63n(1000)))
+	auditID := base52.Encode(time.Now().UnixNano() + rand.Int63n(1000))
+	if customID := os.Getenv(EnvNameAuditID); customID != "" {
+		auditID = fmt.Sprintf("%s_%s", auditID, customID)
+	}
+	fname := filepath.Join(dir, auditID)
 	f, err := os.Create(fname)
 	if err != nil {
 		return errors.Annotate(err, "create audit log")
@@ -183,7 +192,11 @@ func ShowAuditLog(dir string, auditID string) error {
 
 // decodeAuditID decodes the auditID to unix timestamp
 func decodeAuditID(auditID string) (time.Time, error) {
-	ts, err := base52.Decode(auditID)
+	tsID := auditID
+	if strings.Contains(auditID, "_") {
+		tsID = strings.Split(auditID, "_")[0]
+	}
+	ts, err := base52.Decode(tsID)
 	if err != nil {
 		return time.Time{}, err
 	}