Merge branch 'master' into grafana_config

cmd/root.go

@@ -177,14 +177,10 @@ the latest stable version will be downloaded from the repository.`,
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&repoOpts.SkipVersionCheck, "skip-version-check", "", false, "Skip the strict version check, by default a version must be a valid SemVer string")
-	rootCmd.Flags().StringVarP(&binary, "binary", "B", "", "Print binary path of a specific version of a component `<component>[:version]`\n"+
+	rootCmd.Flags().StringVar(&binary, "binary", "", "Print binary path of a specific version of a component `<component>[:version]`\n"+
 		"and the latest version installed will be selected if no version specified")
 	rootCmd.Flags().StringVarP(&tag, "tag", "T", "", "[Deprecated] Specify a tag for component instance")
 	rootCmd.Flags().StringVar(&binPath, "binpath", "", "Specify the binary path of component instance")
-	// Some components will define themself -h flag, eg:
-	// $ tiup dumpling -h ${host}.
-	// We try to leave the handling of `-h` flag to the component.
-	rootCmd.PersistentFlags().Bool("help", false, "Help for this command")
 	rootCmd.Flags().BoolVarP(&printVersion, "version", "v", false, "Print the version of tiup")
 
 	rootCmd.AddCommand(
@@ -201,18 +197,15 @@ the latest stable version will be downloaded from the repository.`,
 
 	originHelpFunc := rootCmd.HelpFunc()
 	rootCmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
-		if len(args) < 2 {
+		cmd, _, _ = cmd.Root().Find(args)
+		if len(args) < 2 || cmd != rootCmd {
 			originHelpFunc(cmd, args)
 			return
 		}
-		env, err := environment.InitEnv(repoOpts)
-		cmd, n, e := cmd.Root().Find(args)
-		if (cmd == rootCmd || e != nil) && len(n) > 0 && err == nil {
-			externalHelp(env, n[0], n[1:]...)
-		} else {
-			cmd.InitDefaultHelpFlag() // make possible 'help' flag to be shown
-			_ = cmd.Help()
-		}
+
+		env, _ := environment.InitEnv(repoOpts)
+		environment.SetGlobalEnv(env)
+		_ = cmd.RunE(cmd, args)
 	})
 
 	rootCmd.SetHelpCommand(newHelpCmd())

cmd/update.go

@@ -16,6 +16,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/pingcap/tiup/pkg/environment"
 	"github.com/pingcap/tiup/pkg/utils"
@@ -47,6 +48,10 @@ latest version. All other flags will be ignored if the flag --self is given.
 
 			env := environment.GlobalEnv()
 			if self {
+				if err := checkTiUPBinary(env); err != nil {
+					return err
+				}
+
 				originFile := env.LocalPath("bin", "tiup")
 				renameFile := env.LocalPath("bin", "tiup.tmp")
 				if err := os.Rename(originFile, renameFile); err != nil {
@@ -100,3 +105,22 @@ func updateComponents(env *environment.Environment, components []string, nightly
 
 	return env.UpdateComponents(components, nightly, force)
 }
+
+// checkTiUPBinary check if TiUP exists in TiUP_HOME
+func checkTiUPBinary(env *environment.Environment) error {
+	tiUPHomePath, _ := filepath.Abs(env.LocalPath("bin", "tiup"))
+
+	realTiUPPath, err := os.Executable()
+	if err != nil {
+		// Ignore the problem that the execution directory cannot be obtained
+		return nil
+	}
+	realTiUPPath, _ = filepath.Abs(realTiUPPath)
+
+	if utils.IsNotExist(tiUPHomePath) || tiUPHomePath != realTiUPPath {
+		fmt.Printf("Tiup install directory is: %s\n", filepath.Dir(realTiUPPath))
+		return fmt.Errorf("If you used some external package manager to install TiUP (e.g., brew), try upgrade with that")
+	}
+
+	return nil
+}

components/bench/Makefile

@@ -77,8 +77,7 @@ tidy:
 clean:
 	@# Target: run the build cleanup steps
 	@rm -rf bin
-	@rm -rf cover
-	@rm -rf tests/*/{bin/*.test,logs,cover/*.out}
+	@rm -rf tests/*/{bin/*.test,logs}
 
 test: failpoint-enable run-tests failpoint-disable
 	@# Target: run tests with failpoint enabled
@@ -91,8 +90,8 @@ run-tests: unit-test
 # Run tests
 unit-test:
 	@# Target: run the code coverage test phase
-	mkdir -p cover
-	TIUP_HOME=$(shell pwd)/../../tests/tiup $(GOTEST) ./... -covermode=count -coverprofile ../../cover/cov.unit-test.out
+	mkdir -p ../../cover
+	TIUP_HOME=$(shell pwd)/../../tests/tiup $(GOTEST) ./... -covermode=count -coverprofile ../../cover/cov.unit-test.bench.out
 
 race: failpoint-enable
 	@# Target: run race check with failpoint enabled

components/client/Makefile

@@ -77,8 +77,7 @@ tidy:
 clean:
 	@# Target: run the build cleanup steps
 	@rm -rf bin
-	@rm -rf cover
-	@rm -rf tests/*/{bin/*.test,logs,cover/*.out}
+	@rm -rf tests/*/{bin/*.test,logs}
 
 test: failpoint-enable run-tests failpoint-disable
 	@# Target: run tests with failpoint enabled
@@ -91,8 +90,8 @@ run-tests: unit-test
 # Run tests
 unit-test:
 	@# Target: run the code coverage test phase
-	mkdir -p cover
-	TIUP_HOME=$(shell pwd)/../../tests/tiup $(GOTEST) ./... -covermode=count -coverprofile ../../cover/cov.unit-test.out
+	mkdir -p ../../cover
+	TIUP_HOME=$(shell pwd)/../../tests/tiup $(GOTEST) ./... -covermode=count -coverprofile ../../cover/cov.unit-test.client.out
 
 race: failpoint-enable
 	@# Target: run race check with failpoint enabled

components/cluster/command/check.go

@@ -28,22 +28,34 @@ func newCheckCmd() *cobra.Command {
 		IdentityFile: path.Join(utils.UserHome(), ".ssh", "id_rsa"),
 	}
 	cmd := &cobra.Command{
-		Use:   "check <topology.yml | cluster-name>",
+		Use:   "check <topology.yml | cluster-name> [scale-out.yml]",
 		Short: "Perform preflight checks for the cluster.",
 		Long: `Perform preflight checks for the cluster. By default, it checks deploy servers
 before a cluster is deployed, the input is the topology.yaml for the cluster.
 If '--cluster' is set, it will perform checks for an existing cluster, the input
 is the cluster name. Some checks are ignore in this mode, such as port and dir
-conflict checks with other clusters`,
+conflict checks with other clusters
+If you want to check the scale-out topology, please use execute the following command
+'	check <cluster-name> <scale-out.yml> --cluster	'
+it will the new instances `,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if len(args) != 1 {
+			if len(args) != 1 && len(args) != 2 {
 				return cmd.Help()
 			}
+			scaleOutTopo := ""
 
 			if opt.ExistCluster {
 				clusterReport.ID = scrubClusterName(args[0])
 			}
-			return cm.CheckCluster(args[0], opt, gOpt)
+
+			if len(args) == 2 {
+				if !opt.ExistCluster {
+					return cmd.Help()
+				}
+				scaleOutTopo = args[1]
+			}
+
+			return cm.CheckCluster(args[0], scaleOutTopo, opt, gOpt)
 		},
 	}
 

components/cluster/command/start.go

@@ -14,12 +14,23 @@
 package command
 
 import (
+	"database/sql"
+	"fmt"
+
+	"github.com/fatih/color"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/cluster/task"
+	"github.com/pingcap/tiup/pkg/crypto/rand"
+	"github.com/pingcap/tiup/pkg/proxy"
 	"github.com/spf13/cobra"
+
+	// for sql/driver
+	_ "github.com/go-sql-driver/mysql"
 )
 
 func newStartCmd() *cobra.Command {
+	var initPasswd bool
+
 	cmd := &cobra.Command{
 		Use:   "start <cluster-name>",
 		Short: "Start a TiDB cluster",
@@ -36,19 +47,84 @@ func newStartCmd() *cobra.Command {
 			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
-			return cm.StartCluster(clusterName, gOpt, func(b *task.Builder, metadata spec.Metadata) {
+			if err := cm.StartCluster(clusterName, gOpt, func(b *task.Builder, metadata spec.Metadata) {
 				b.UpdateTopology(
 					clusterName,
 					tidbSpec.Path(clusterName),
 					metadata.(*spec.ClusterMeta),
 					nil, /* deleteNodeIds */
 				)
-			})
+			}); err != nil {
+				return err
+			}
+
+			// init password
+			if initPasswd {
+				pwd, err := initPassword(clusterName)
+				if err != nil {
+					log.Errorf("failed to set root password of TiDB database to '%s'", pwd)
+					return err
+				}
+				log.Warnf("The root password of TiDB database has been changed.")
+				fmt.Printf("The new password is: '%s'.", color.HiYellowString(pwd)) // use fmt to avoid printing to audit log
+				log.Warnf("Copy and record it to somewhere safe, %s, and will not be stored.", color.HiRedString("it is only displayed once"))
+				log.Warnf("The generated password %s.", color.HiRedString("could NOT be get and shown again"))
+			}
+			return nil
 		},
 	}
 
+	cmd.Flags().BoolVar(&initPasswd, "init", false, "Initialize a secure root password for the database")
 	cmd.Flags().StringSliceVarP(&gOpt.Roles, "role", "R", nil, "Only start specified roles")
 	cmd.Flags().StringSliceVarP(&gOpt.Nodes, "node", "N", nil, "Only start specified nodes")
 
 	return cmd
 }
+
+func initPassword(clusterName string) (string, error) {
+	metadata, err := spec.ClusterMetadata(clusterName)
+	if err != nil {
+		return "", err
+	}
+	tcpProxy := proxy.GetTCPProxy()
+
+	// generate password
+	pwd, err := rand.Password(18)
+	if err != nil {
+		return pwd, err
+	}
+
+	var lastErr error
+	for _, spec := range metadata.Topology.TiDBServers {
+		spec := spec
+		endpoint := fmt.Sprintf("%s:%d", spec.Host, spec.Port)
+		if tcpProxy != nil {
+			closeC := tcpProxy.Run([]string{endpoint})
+			defer tcpProxy.Close(closeC)
+			endpoint = tcpProxy.GetEndpoints()[0]
+		}
+		db, err := createDB(endpoint)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		defer db.Close()
+
+		sqlStr := fmt.Sprintf("SET PASSWORD FOR 'root'@'%%' = '%s'; FLUSH PRIVILEGES;", pwd)
+		_, err = db.Exec(sqlStr)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		return pwd, nil
+	}
+
+	return pwd, lastErr
+}
+
+func createDB(endpoint string) (db *sql.DB, err error) {
+	dsn := fmt.Sprintf("root:@tcp(%s)/?charset=utf8mb4,utf8&multiStatements=true", endpoint)
+	db, err = sql.Open("mysql", dsn)
+
+	return
+}

components/cluster/command/test.go

@@ -15,7 +15,6 @@ package command
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 	"fmt"
 
@@ -72,13 +71,6 @@ func newTestCmd() *cobra.Command {
 	return cmd
 }
 
-func createDB(endpoint string) (db *sql.DB, err error) {
-	dsn := fmt.Sprintf("root:@tcp(%s)/?charset=utf8mb4,utf8&multiStatements=true", endpoint)
-	db, err = sql.Open("mysql", dsn)
-
-	return
-}
-
 // To check if test.ti_cluster has data
 func data(topo *spec.Specification, tcpProxy *proxy.TCPProxy) error {
 	errg, _ := errgroup.WithContext(context.Background())

embed/examples/dm/topology.example.yaml

@@ -47,6 +47,9 @@ worker_servers:
     # # Config is used to overwrite the `server_configs.dm-worker` values
     config:
       log-level: info
+      # keepalive-ttl: 60
+      # relay-keepalive-ttl: 1800 # since v2.0.2
+      # relay-dir: "" # since v5.4.0
   - host: 10.0.1.19
 
 monitoring_servers:

embed/templates/scripts/run_tidb.sh.tpl

@@ -27,6 +27,9 @@ exec env GODEBUG=madvdontneed=1 bin/tidb-server \
     --host="{{.ListenHost}}" \
     --advertise-address="{{.AdvertiseAddr}}" \
     --store="tikv" \
+{{- if .SupportSecboot}}
+    --initialize-insecure \
+{{- end}}
     --path="{{template "PDList" .Endpoints}}" \
     --log-slow-query="{{.LogDir}}/tidb_slow_query.log" \
     --config=conf/tidb.toml \

examples

@@ -1 +1 @@
-embed/templates/examples
+embed/examples

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/r3labs/diff/v2 v2.14.0
 	github.com/relex/aini v1.5.0
 	github.com/sergi/go-diff v1.2.0
+	github.com/sethvargo/go-password v0.2.0
 	github.com/shirou/gopsutil v3.21.8+incompatible
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/spf13/cobra v1.2.1

go.sum

@@ -483,6 +483,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI=
+github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE=
 github.com/shirou/gopsutil v3.21.8+incompatible h1:sh0foI8tMRlCidUJR+KzqWYWxrkuuPIGiO6Vp+KXdCU=
 github.com/shirou/gopsutil v3.21.8+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=