Merge branch 'master' into fix-default-value

.github/workflows/integrate-cluster-cmd.yaml

@@ -84,20 +84,24 @@ jobs:
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
           ls ${{ env.working-directory }}
-          tar czvf ${{ env.working-directory }}/logs.tar.gz ${{ env.working-directory }}/logs/
+
+      - name: Detect error log
+        working-directory: ${{ env.working-directory }}
+        # if: always()
+        run: |
+          bash ./tests/tiup-cluster/script/detect_error.sh ./logs/
 
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: component_logs
-          path: ${{ env.working-directory }}/logs.tar.gz
+          path: ${{ env.working-directory }}/logs/
 
       - name: Output cluster debug log
         working-directory: ${{ env.working-directory }}

.github/workflows/integrate-cluster-scale.yaml

@@ -84,20 +84,24 @@ jobs:
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
           ls ${{ env.working-directory }}
-          tar czvf ${{ env.working-directory }}/logs.tar.gz ${{ env.working-directory }}/logs/
+
+      - name: Detect error log
+        working-directory: ${{ env.working-directory }}
+        # if: always()
+        run: |
+          bash ./tests/tiup-cluster/script/detect_error.sh ./logs/
 
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: cluster_logs
-          path: ${{ env.working-directory }}/logs.tar.gz
+          path: ${{ env.working-directory }}/logs/
 
       - name: Output cluster debug log
         working-directory: ${{ env.working-directory }}

CHANGELOG.md

@@ -1,8 +1,27 @@
 TiUP Changelog
 
+## [1.9.6] 2022-05-20
+
+### Fix
+
+- Fix incorrect output of `display` in certain circumstances for `tiup-cluster` ([#1889](https://github.com/pingcap/tiup/pull/1889), [@srstack](https://github.com/srstack))
+
+## [1.9.5] 2022-05-10
+
+### Fixes
+
+- Fix `prune` incorrectly destroy pump/drainer node before they become `Tombstone` in `tiup-cluster` ([#1851](https://github.com/pingcap/tiup/pull/1851), [@srstack](https://github.com/srstack))
+- Report error when multiple pump nodes with the same `ip:port` found in `tiup-cluster` ([#1856](https://github.com/pingcap/tiup/pull/1856), [@srstack](https://github.com/srstack))
+- Get node status of pump/drainer from PD in `tiup-cluster` ([#1862](https://github.com/pingcap/tiup/pull/1862), [@srstack](https://github.com/srstack))
+
+### Improvements
+
+- Check node status concurrently and support custom timeout for `display` in `tiup-cluster` ([#1867](https://github.com/pingcap/tiup/pull/1867), [@srstack](https://github.com/srstack))
+- Support `tidb-lightning` in `tiup-ctl` ([#1863](https://github.com/pingcap/tiup/pull/1863), [@nexustar](https://github.com/nexustar))
+
 ## [1.9.4] 2022-04-12
 
-## Fixes
+### Fixes
 
 - Fix copy error when file is read only in `tiup-playground` ([#1816](https://github.com/pingcap/tiup/pull/1816), [@breeswish](https://github.com/breeswish))
 - Fix `data-dir` not properly handled for TiCDC v6.0.0 in `tiup-cluster` ([#1838](https://github.com/pingcap/tiup/pull/1838), [@overvenus](https://github.com/overvenus))

cmd/root.go

@@ -142,17 +142,25 @@ the latest stable version will be downloaded from the repository.`,
 				return nil
 			case "--binpath":
 				if len(args) < 2 {
-					return fmt.Errorf("flag needs an argument: %s", args[0])
+					return fmt.Errorf("flag %s needs an argument", args[0])
 				}
 				binPath = args[1]
 				args = args[2:]
 			case "--tag", "-T":
 				if len(args) < 2 {
-					return fmt.Errorf("flag needs an argument: %s", args[0])
+					return fmt.Errorf("flag %s needs an argument", args[0])
 				}
 				tag = args[1]
 				args = args[2:]
 			}
+
+			// component may use tag from environment variable. as workaround, make tiup set the same tag
+			for i := 0; i < len(args)-1; i++ {
+				if args[i] == "--tag" || args[i] == "-T" {
+					tag = args[i+1]
+				}
+			}
+
 			if len(args) < 1 {
 				return cmd.Help()
 			}

components/cluster/command/clean.go

@@ -57,6 +57,14 @@ You can retain some nodes and roles data when cleanup the cluster, eg:
 
 			return cm.CleanCluster(clusterName, gOpt, cleanOpt, skipConfirm)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringArrayVar(&cleanOpt.RetainDataNodes, "ignore-node", nil, "Specify the nodes or hosts whose data will be retained")

components/cluster/command/deploy.go

@@ -67,6 +67,14 @@ func newDeploy() *cobra.Command {
 
 			return cm.Deploy(clusterName, version, topoFile, opt, postDeployHook, skipConfirm, gOpt)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 2:
+				return nil, cobra.ShellCompDirectiveDefault
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringVarP(&opt.User, "user", "u", utils.CurrentUser(), "The user name to login via SSH. The user must has root (or sudo) privilege.")

components/cluster/command/destroy.go

@@ -53,6 +53,14 @@ You can retain some nodes and roles data when destroy cluster, eg:
 
 			return cm.DestroyCluster(clusterName, gOpt, destroyOpt, skipConfirm)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringArrayVar(&destroyOpt.RetainDataNodes, "retain-node-data", nil, "Specify the nodes or hosts whose data will be retained")

components/cluster/command/disable.go

@@ -36,6 +36,14 @@ func newDisableCmd() *cobra.Command {
 
 			return cm.EnableCluster(clusterName, gOpt, false)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringSliceVarP(&gOpt.Roles, "role", "R", nil, "Only disable specified roles")

components/cluster/command/display.go

@@ -16,8 +16,11 @@ package command
 import (
 	"errors"
 	"fmt"
+	"strings"
+	"time"
 
 	perrs "github.com/pingcap/errors"
+	"github.com/pingcap/tiup/pkg/cluster/manager"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/spf13/cobra"
@@ -29,6 +32,7 @@ func newDisplayCmd() *cobra.Command {
 		showDashboardOnly bool
 		showVersionOnly   bool
 		showTiKVLabels    bool
+		statusTimeout     uint64
 	)
 	cmd := &cobra.Command{
 		Use:   "display <cluster-name>",
@@ -38,6 +42,7 @@ func newDisplayCmd() *cobra.Command {
 				return cmd.Help()
 			}
 
+			gOpt.APITimeout = statusTimeout
 			clusterName = args[0]
 			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
@@ -67,13 +72,21 @@ func newDisplayCmd() *cobra.Command {
 				if err != nil {
 					return err
 				}
-				return cm.DisplayDashboardInfo(clusterName, tlsCfg)
+				return cm.DisplayDashboardInfo(clusterName, time.Second*time.Duration(gOpt.APITimeout), tlsCfg)
 			}
 			if showTiKVLabels {
 				return cm.DisplayTiKVLabels(clusterName, gOpt)
 			}
 			return cm.Display(clusterName, gOpt)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringSliceVarP(&gOpt.Roles, "role", "R", nil, "Only display specified roles")
@@ -82,6 +95,18 @@ func newDisplayCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&showDashboardOnly, "dashboard", false, "Only display TiDB Dashboard information")
 	cmd.Flags().BoolVar(&showVersionOnly, "version", false, "Only display TiDB cluster version")
 	cmd.Flags().BoolVar(&showTiKVLabels, "labels", false, "Only display labels of specified TiKV role or nodes")
+	cmd.Flags().Uint64Var(&statusTimeout, "status-timeout", 10, "Timeout in seconds when getting node status")
 
 	return cmd
 }
+
+func shellCompGetClusterName(cm *manager.Manager, toComplete string) ([]string, cobra.ShellCompDirective) {
+	var result []string
+	clusters, _ := cm.GetClusterList()
+	for _, c := range clusters {
+		if strings.HasPrefix(c.Name, toComplete) {
+			result = append(result, c.Name)
+		}
+	}
+	return result, cobra.ShellCompDirectiveNoFileComp
+}

components/cluster/command/edit_config.go

@@ -34,6 +34,14 @@ func newEditConfigCmd() *cobra.Command {
 
 			return cm.EditConfig(clusterName, opt, skipConfirm)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringVarP(&opt.NewTopoFile, "topology-file", "", opt.NewTopoFile, "Use provided topology file to substitute the original one instead of editing it.")

components/cluster/command/enable.go

@@ -36,6 +36,14 @@ func newEnableCmd() *cobra.Command {
 
 			return cm.EnableCluster(clusterName, gOpt, true)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringSliceVarP(&gOpt.Roles, "role", "R", nil, "Only enable specified roles")

components/cluster/command/exec.go

@@ -35,6 +35,14 @@ func newExecCmd() *cobra.Command {
 
 			return cm.Exec(clusterName, opt, gOpt)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringVar(&opt.Command, "command", "ls", "the command run on cluster host")

components/cluster/command/prune.go

@@ -32,6 +32,14 @@ func newPruneCmd() *cobra.Command {
 
 			return cm.DestroyTombstone(clusterName, gOpt, skipConfirm)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().BoolVar(&gOpt.Force, "force", false, "Ignore errors when deleting the instance with data from the cluster")

components/cluster/command/reload.go

@@ -39,6 +39,14 @@ func newReloadCmd() *cobra.Command {
 
 			return cm.Reload(clusterName, gOpt, skipRestart, skipConfirm)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().BoolVar(&gOpt.Force, "force", false, "Force reload without transferring PD leader and ignore remote error")

components/cluster/command/restart.go

@@ -36,6 +36,14 @@ func newRestartCmd() *cobra.Command {
 
 			return cm.RestartCluster(clusterName, gOpt, skipConfirm)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringSliceVarP(&gOpt.Roles, "role", "R", nil, "Only restart specified roles")

components/cluster/command/scale_in.go

@@ -50,6 +50,14 @@ func newScaleInCmd() *cobra.Command {
 
 			return cm.ScaleIn(clusterName, skipConfirm, gOpt, scale)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringSliceVarP(&gOpt.Nodes, "node", "N", nil, "Specify the nodes (required)")

components/cluster/command/scale_out.go

@@ -76,6 +76,16 @@ func newScaleOutCmd() *cobra.Command {
 				gOpt,
 			)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			case 1:
+				return nil, cobra.ShellCompDirectiveDefault
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().StringVarP(&opt.User, "user", "u", utils.CurrentUser(), "The user name to login via SSH. The user must has root (or sudo) privilege.")

components/cluster/command/show_config.go

@@ -32,6 +32,14 @@ func newShowConfigCmd() *cobra.Command {
 
 			return cm.ShowConfig(clusterName)
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	return cmd

components/cluster/command/start.go

@@ -80,6 +80,14 @@ func newStartCmd() *cobra.Command {
 			}
 			return nil
 		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			switch len(args) {
+			case 0:
+				return shellCompGetClusterName(cm, toComplete)
+			default:
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+		},
 	}
 
 	cmd.Flags().BoolVar(&initPasswd, "init", false, "Initialize a secure root password for the database")