Merge branch 'master' into issue-1865

.github/workflows/integrate-cluster-cmd.yaml

@@ -84,20 +84,24 @@ jobs:
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
           ls ${{ env.working-directory }}
-          tar czvf ${{ env.working-directory }}/logs.tar.gz ${{ env.working-directory }}/logs/
+
+      - name: Detect error log
+        working-directory: ${{ env.working-directory }}
+        # if: always()
+        run: |
+          bash ./tests/tiup-cluster/script/detect_error.sh ./logs/
 
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: component_logs
-          path: ${{ env.working-directory }}/logs.tar.gz
+          path: ${{ env.working-directory }}/logs/
 
       - name: Output cluster debug log
         working-directory: ${{ env.working-directory }}

.github/workflows/integrate-cluster-scale.yaml

@@ -84,20 +84,24 @@ jobs:
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
           ls ${{ env.working-directory }}
-          tar czvf ${{ env.working-directory }}/logs.tar.gz ${{ env.working-directory }}/logs/
+
+      - name: Detect error log
+        working-directory: ${{ env.working-directory }}
+        # if: always()
+        run: |
+          bash ./tests/tiup-cluster/script/detect_error.sh ./logs/
 
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: cluster_logs
-          path: ${{ env.working-directory }}/logs.tar.gz
+          path: ${{ env.working-directory }}/logs/
 
       - name: Output cluster debug log
         working-directory: ${{ env.working-directory }}

CHANGELOG.md

@@ -1,5 +1,11 @@
 TiUP Changelog
 
+## [1.9.6] 2022-05-20
+
+### Fix
+
+- Fix incorrect output of `display` in certain circumstances for `tiup-cluster` ([#1889](https://github.com/pingcap/tiup/pull/1889), [@srstack](https://github.com/srstack))
+
 ## [1.9.5] 2022-05-10
 
 ### Fixes

components/dm/command/deploy.go

@@ -22,10 +22,10 @@ import (
 	operator "github.com/pingcap/tiup/pkg/cluster/operation"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/cluster/task"
+	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/tui"
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/spf13/cobra"
-	"golang.org/x/mod/semver"
 )
 
 func newDeployCmd() *cobra.Command {
@@ -69,12 +69,7 @@ func newDeployCmd() *cobra.Command {
 }
 
 func supportVersion(vs string) error {
-	if !semver.IsValid(vs) {
-		return nil
-	}
-
-	majorMinor := semver.MajorMinor(vs)
-	if semver.Compare(majorMinor, "v2.0") < 0 {
+	if !tidbver.DMSupportDeploy(vs) {
 		return errors.Errorf("Only support version not less than v2.0")
 	}
 

components/dm/command/deploy_test.go

@@ -10,12 +10,10 @@ func TestSupportVersion(t *testing.T) {
 	assert := require.New(t)
 
 	tests := map[string]bool{ // version to support or not
-		"v2.0.0":        true,
-		"v3.0.0":        true,
-		"v2.0.0-beta.1": true,
-		"v2.0.0-alpha":  true,
-		"v1.0.1":        false,
-		"v1.1.1":        false,
+		"v2.0.0": true,
+		"v6.0.0": true,
+		"v1.0.1": false,
+		"v1.1.1": false,
 	}
 
 	for v, support := range tests {

components/playground/instance/ticdc.go

@@ -20,8 +20,8 @@ import (
 	"strings"
 
 	tiupexec "github.com/pingcap/tiup/pkg/exec"
+	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/utils"
-	"golang.org/x/mod/semver"
 )
 
 // TiCDC represent a ticdc instance.
@@ -62,9 +62,8 @@ func (c *TiCDC) Start(ctx context.Context, version utils.Version) error {
 		fmt.Sprintf("--log-file=%s", c.LogFile()),
 	}
 	clusterVersion := string(version)
-	if semver.Compare(clusterVersion, "v4.0.13") >= 0 {
-		if (semver.Major(clusterVersion) == "v4" && semver.Compare(clusterVersion, "v4.0.14") >= 0) ||
-			(semver.Major(clusterVersion) == "v5" && semver.Compare(clusterVersion, "v5.0.3") >= 0) {
+	if tidbver.TiCDCSupportConfigFile(clusterVersion) {
+		if tidbver.TiCDCSupportDataDir(clusterVersion) {
 			args = append(args, fmt.Sprintf("--data-dir=%s", filepath.Join(c.Dir, "data")))
 		} else {
 			args = append(args, fmt.Sprintf("--sort-dir=%s/tmp/sorter", filepath.Join(c.Dir, "data")))

components/playground/instance/tiflash_config.go

@@ -18,8 +18,8 @@ import (
 	"io"
 	"strings"
 
+	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/utils"
-	"golang.org/x/mod/semver"
 )
 
 const tiflashDaemonConfig = `
@@ -104,7 +104,7 @@ func writeTiFlashConfig(w io.Writer, version utils.Version, tcpPort, httpPort, s
 	logDir := fmt.Sprintf("%s/log", deployDir)
 	ip := AdvertiseHost(host)
 	var conf string
-	if semver.Compare(version.String(), "v5.4.0") >= 0 || version.IsNightly() {
+	if tidbver.TiFlashNotNeedSomeConfig(version.String()) {
 		conf = fmt.Sprintf(tiflashConfig, pdAddrs, httpPort, tcpPort,
 			deployDir, dataDir, tmpDir, logDir, servicePort, metricsPort,
 			ip, strings.Join(tidbStatusAddrs, ","), clusterManagerPath, "")

components/playground/instance/tiflash_proxy_config.go

@@ -17,8 +17,8 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/utils"
-	"golang.org/x/mod/semver"
 )
 
 const tiflashProxyConfig = `
@@ -52,7 +52,7 @@ func writeTiFlashProxyConfig(w io.Writer, version utils.Version, host, deployDir
 	logDir := fmt.Sprintf("%s/log", deployDir)
 	ip := AdvertiseHost(host)
 	var statusAddr string
-	if semver.Compare(version.String(), "v4.0.5") >= 0 || version.IsNightly() {
+	if tidbver.TiFlashSupportAdvertiseStatusAddr(version.String()) {
 		statusAddr = fmt.Sprintf(`status-addr = "0.0.0.0:%[2]d"
 advertise-status-addr = "%[1]s:%[2]d"`, ip, proxyStatusPort)
 	} else {

embed/examples/dm/minimal.yaml

@@ -26,9 +26,6 @@ monitoring_servers:
 
 grafana_servers:
   - host: 172.19.0.101
-  # config:       #  Enable this part if you want to use WebUI, make sure tiup dm -v newer than v1.9.0.
-    # auth.anonymous.enabled: true
-    # security.allow_embedding: true
 
 alertmanager_servers:
   - host: 172.19.0.101

embed/examples/dm/topology.example.yaml

@@ -68,9 +68,6 @@ grafana_servers:
     # deploy_dir: /tidb-deploy/grafana-3000
     # grafana dashboard dir on TiUP machine
     # dashboard_dir: /home/tidb/dashboards
-    # config:       # Enable this part if you want to use WebUI, make sure tiup dm -v newer than v1.9.0.
-    #   auth.anonymous.enabled: true
-    #   security.allow_embedding: true
 
 alertmanager_servers:
   - host: 10.0.1.15

pkg/cluster/manager/basic.go

@@ -31,9 +31,9 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/task"
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/pingcap/tiup/pkg/set"
+	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/tui"
 	"github.com/pingcap/tiup/pkg/utils"
-	"golang.org/x/mod/semver"
 )
 
 // EnableCluster enable/disable the service in a cluster
@@ -296,7 +296,7 @@ func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) {
 func checkTiFlashWithTLS(topo spec.Topology, version string) error {
 	if clusterSpec, ok := topo.(*spec.Specification); ok {
 		if clusterSpec.GlobalOptions.TLSEnabled {
-			if (semver.Compare(version, "v4.0.5") < 0 &&
+			if (!tidbver.TiFlashSupportTLS(version) &&
 				len(clusterSpec.TiFlashServers) > 0) &&
 				version != utils.NightlyVersionAlias {
 				return fmt.Errorf("TiFlash %s is not supported in TLS enabled cluster", version)

pkg/cluster/manager/display.go

@@ -24,6 +24,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/fatih/color"
@@ -362,12 +363,16 @@ func (m *Manager) DisplayTiKVLabels(name string, opt operator.Options) error {
 	if err != nil {
 		return err
 	}
+
+	var mu sync.Mutex
 	topo.IterInstance(func(ins spec.Instance) {
 		if ins.ComponentName() == spec.ComponentPD {
 			status := ins.Status(ctx, statusTimeout, tlsCfg, masterList...)
 			if strings.HasPrefix(status, "Up") || strings.HasPrefix(status, "Healthy") {
 				instAddr := fmt.Sprintf("%s:%d", ins.GetHost(), ins.GetPort())
+				mu.Lock()
 				masterActive = append(masterActive, instAddr)
+				mu.Unlock()
 			}
 		}
 	}, opt.Concurrency)
@@ -482,6 +487,7 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI
 	masterActive := make([]string, 0)
 	masterStatus := make(map[string]string)
 
+	var mu sync.Mutex
 	topo.IterInstance(func(ins spec.Instance) {
 		if ins.ComponentName() != spec.ComponentPD && ins.ComponentName() != spec.ComponentDMMaster {
 			return
@@ -490,7 +496,9 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI
 		status := ins.Status(ctx, statusTimeout, tlsCfg, masterList...)
 		if strings.HasPrefix(status, "Up") || strings.HasPrefix(status, "Healthy") {
 			instAddr := fmt.Sprintf("%s:%d", ins.GetHost(), ins.GetPort())
+			mu.Lock()
 			masterActive = append(masterActive, instAddr)
+			mu.Unlock()
 		}
 		masterStatus[ins.ID()] = status
 	}, opt.Concurrency)
@@ -564,6 +572,7 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI
 		if ins.IsPatched() {
 			roleName += " (patched)"
 		}
+		mu.Lock()
 		clusterInstInfos = append(clusterInstInfos, InstInfo{
 			ID:            ins.ID(),
 			Role:          roleName,
@@ -577,6 +586,7 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI
 			Port:          ins.GetPort(),
 			Since:         since,
 		})
+		mu.Unlock()
 	}, opt.Concurrency)
 
 	// Sort by role,host,ports

pkg/cluster/spec/cdc.go

@@ -24,7 +24,7 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
 	"github.com/pingcap/tiup/pkg/meta"
-	"golang.org/x/mod/semver"
+	"github.com/pingcap/tiup/pkg/tidbver"
 )
 
 // CDCSpec represents the Drainer topology specification in topology.yaml
@@ -163,7 +163,7 @@ func (i *CDCInstance) InitConfig(
 	globalConfig := topo.ServerConfigs.CDC
 	instanceConfig := spec.Config
 
-	if semver.Compare(clusterVersion, "v4.0.13") == -1 {
+	if !tidbver.TiCDCSupportConfigFile(clusterVersion) {
 		if len(globalConfig)+len(instanceConfig) > 0 {
 			return perrs.New("server_config is only supported with TiCDC version v4.0.13 or later")
 		}

pkg/cluster/spec/profile.go

@@ -59,14 +59,19 @@ var initialized = false
 // The directory will be created before return if it does not already exist.
 func Initialize(base string) error {
 	tiupData := os.Getenv(tiuplocaldata.EnvNameComponentDataDir)
-	if tiupData == "" {
+	tiupHome := os.Getenv(tiuplocaldata.EnvNameHome)
+
+	switch {
+	case tiupData != "":
+		profileDir = tiupData
+	case tiupHome != "":
+		profileDir = path.Join(tiupHome, tiuplocaldata.StorageParentDir, base)
+	default:
 		homeDir, err := getHomeDir()
 		if err != nil {
 			return errors.Trace(err)
 		}
 		profileDir = path.Join(homeDir, ".tiup", tiuplocaldata.StorageParentDir, base)
-	} else {
-		profileDir = tiupData
 	}
 
 	clusterBaseDir := filepath.Join(profileDir, TiUPClusterDir)

pkg/cluster/spec/spec.go

@@ -31,11 +31,10 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/pingcap/tiup/pkg/proxy"
+	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/tui"
-	"github.com/pingcap/tiup/pkg/utils"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.uber.org/zap"
-	"golang.org/x/mod/semver"
 )
 
 const (
@@ -413,12 +412,12 @@ func (s *Specification) GetPDList() []string {
 // AdjustByVersion modify the spec by cluster version.
 func (s *Specification) AdjustByVersion(clusterVersion string) {
 	// CDC does not support data dir for version below v4.0.13, and also v5.0.0-rc, set it to empty.
-	if semver.Compare(clusterVersion, "v4.0.13") == -1 || clusterVersion == "v5.0.0-rc" {
+	if !tidbver.TiCDCSupportConfigFile(clusterVersion) {
 		for _, server := range s.CDCServers {
 			server.DataDir = ""
 		}
 	}
-	if semver.Compare(clusterVersion, "v5.4.0") >= 0 || strings.Contains(clusterVersion, utils.NightlyVersionAlias) {
+	if tidbver.NgMonitorDeployByDefault(clusterVersion) {
 		for _, m := range s.Monitors {
 			if m.NgPort == 0 {
 				m.NgPort = 12020

pkg/cluster/spec/spec_test.go

@@ -740,8 +740,6 @@ cdc_servers:
 		"v5.1.0": {true, true, true},
 
 		"v5.0.0-rc":    {false, false, false},
-		"v5.1.0-alpha": {true, true, false},
-		"v5.2.0-alpha": {true, true, false},
 		"v6.0.0-alpha": {true, true, true},
 		"v6.1.0":       {true, true, true},
 		"v99.0.0":      {true, true, true},

pkg/cluster/spec/tidb.go

@@ -24,7 +24,7 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
 	"github.com/pingcap/tiup/pkg/meta"
-	"golang.org/x/mod/semver"
+	"github.com/pingcap/tiup/pkg/tidbver"
 )
 
 // TiDBSpec represents the TiDB topology specification in topology.yaml
@@ -146,7 +146,7 @@ func (i *TiDBInstance) InitConfig(
 		AppendEndpoints(topo.Endpoints(deployUser)...).
 		WithListenHost(i.GetListenHost()).
 		WithAdvertiseAddr(spec.Host).
-		SupportSecureBootstrap(semver.Compare(clusterVersion, "v5.3.0") >= 0)
+		SupportSecureBootstrap(tidbver.TiDBSupportSecureBoot(clusterVersion))
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_tidb_%s_%d.sh", i.GetHost(), i.GetPort()))
 	if err := cfg.ConfigToFile(fp); err != nil {
 		return err