Merge branch 'fix-import-binlog' of github.com:lucklove/tiup into fix…

…-import-binlog

components/cluster/command/patch.go

@@ -41,7 +41,7 @@ func newPatchCmd() *cobra.Command {
 			clusterName := args[0]
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
-			return cm.Patch(clusterName, args[1], gOpt, overwrite, offlineMode)
+			return cm.Patch(clusterName, args[1], gOpt, overwrite, offlineMode, skipConfirm)
 		},
 	}
 

components/cluster/command/reload.go

@@ -36,7 +36,7 @@ func newReloadCmd() *cobra.Command {
 			clusterName := args[0]
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
-			return cm.Reload(clusterName, gOpt, skipRestart)
+			return cm.Reload(clusterName, gOpt, skipRestart, skipConfirm)
 		},
 	}
 

components/cluster/command/rename.go

@@ -34,7 +34,7 @@ func newRenameCmd() *cobra.Command {
 			newClusterName := args[1]
 			teleCommand = append(teleCommand, scrubClusterName(oldClusterName))
 
-			return cm.Rename(oldClusterName, gOpt, newClusterName)
+			return cm.Rename(oldClusterName, gOpt, newClusterName, skipConfirm)
 		},
 	}
 

components/cluster/command/restart.go

@@ -33,7 +33,7 @@ func newRestartCmd() *cobra.Command {
 			clusterName := args[0]
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
-			return cm.RestartCluster(clusterName, gOpt)
+			return cm.RestartCluster(clusterName, gOpt, skipConfirm)
 		},
 	}
 

components/cluster/command/stop.go

@@ -33,7 +33,7 @@ func newStopCmd() *cobra.Command {
 			clusterName := args[0]
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
-			return cm.StopCluster(clusterName, gOpt)
+			return cm.StopCluster(clusterName, gOpt, skipConfirm)
 		},
 	}
 

components/dm/command/patch.go

@@ -41,7 +41,7 @@ func newPatchCmd() *cobra.Command {
 
 			clusterName := args[0]
 
-			return cm.Patch(clusterName, args[1], gOpt, overwrite, offlineMode)
+			return cm.Patch(clusterName, args[1], gOpt, overwrite, offlineMode, skipConfirm)
 		},
 	}
 

components/dm/command/reload.go

@@ -35,7 +35,7 @@ func newReloadCmd() *cobra.Command {
 
 			clusterName := args[0]
 
-			return cm.Reload(clusterName, gOpt, skipRestart)
+			return cm.Reload(clusterName, gOpt, skipRestart, skipConfirm)
 		},
 	}
 

components/dm/command/restart.go

@@ -28,7 +28,7 @@ func newRestartCmd() *cobra.Command {
 
 			clusterName := args[0]
 
-			return cm.RestartCluster(clusterName, gOpt)
+			return cm.RestartCluster(clusterName, gOpt, skipConfirm)
 		},
 	}
 

components/dm/command/stop.go

@@ -28,7 +28,7 @@ func newStopCmd() *cobra.Command {
 
 			clusterName := args[0]
 
-			return cm.StopCluster(clusterName, gOpt)
+			return cm.StopCluster(clusterName, gOpt, skipConfirm)
 		},
 	}
 

components/playground/instance/process.go

@@ -2,6 +2,7 @@ package instance
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -84,8 +85,8 @@ func (p *process) Cmd() *exec.Cmd {
 	return p.cmd
 }
 
-// NewComponentProcess create a Process instance.
-func NewComponentProcess(ctx context.Context, dir, binPath, component string, version utils.Version, arg ...string) (Process, error) {
+// NewComponentProcessWithEnvs create a Process instance with given environment variables.
+func NewComponentProcessWithEnvs(ctx context.Context, dir, binPath, component string, version utils.Version, envs map[string]string, arg ...string) (Process, error) {
 	if dir == "" {
 		panic("dir must be set")
 	}
@@ -95,20 +96,31 @@ func NewComponentProcess(ctx context.Context, dir, binPath, component string, ve
 
 	env := environment.GlobalEnv()
 	params := &tiupexec.PrepareCommandParams{
-		Ctx:         ctx,
-		Component:   component,
-		Version:     version,
-		BinPath:     binPath,
-		InstanceDir: dir,
-		WD:          dir,
-		Args:        arg,
-		SysProcAttr: SysProcAttr,
-		Env:         env,
+		Ctx:          ctx,
+		Component:    component,
+		Version:      version,
+		BinPath:      binPath,
+		InstanceDir:  dir,
+		WD:           dir,
+		Args:         arg,
+		EnvVariables: make([]string, 0),
+		SysProcAttr:  SysProcAttr,
+		Env:          env,
 	}
+	for k, v := range envs {
+		pair := fmt.Sprintf("%s=%s", k, v)
+		params.EnvVariables = append(params.EnvVariables, pair)
+	}
+
 	cmd, err := tiupexec.PrepareCommand(params)
 	if err != nil {
 		return nil, err
 	}
 
 	return &process{cmd: cmd}, nil
 }
+
+// NewComponentProcess create a Process instance.
+func NewComponentProcess(ctx context.Context, dir, binPath, component string, version utils.Version, arg ...string) (Process, error) {
+	return NewComponentProcessWithEnvs(ctx, dir, binPath, component, version, nil, arg...)
+}

components/playground/instance/tikv.go

@@ -71,7 +71,9 @@ func (inst *TiKVInstance) Start(ctx context.Context, version utils.Version) erro
 	}
 
 	var err error
-	if inst.Process, err = NewComponentProcess(ctx, inst.Dir, inst.BinPath, "tikv", version, args...); err != nil {
+	envs := make(map[string]string)
+	envs["MALLOC_CONF"] = "prof:true,prof_active:true,prof.active:false"
+	if inst.Process, err = NewComponentProcessWithEnvs(ctx, inst.Dir, inst.BinPath, "tikv", version, envs, args...); err != nil {
 		return err
 	}
 	logIfErr(inst.Process.SetOutputFile(inst.LogFile()))

embed/templates/config/grafana.ini.tpl

@@ -17,7 +17,7 @@ data = {{.DeployDir}}/data
 #
 # Directory where grafana can store logs
 #
-logs = {{.DeployDir}}/logs
+logs = {{.DeployDir}}/log
 #
 # Directory where grafana will automatically scan and look for plugins
 #
@@ -289,4 +289,4 @@ path = {{.DeployDir}}/dashboards
 #################################### Internal Grafana Metrics ##########################
 # Url used to to import dashboards directly from Grafana.net
 [grafana_net]
-url = https://grafana.net
+url = https://grafana.net

embed/templates/scripts/run_pd.sh.tpl

@@ -10,9 +10,9 @@ cd "${DEPLOY_DIR}" || exit 1
 {{- define "PDList"}}
   {{- range $idx, $pd := .}}
     {{- if eq $idx 0}}
-      {{- $pd.Name}}={{$pd.Scheme}}://{{$pd.IP}}:{{$pd.PeerPort}}
+      {{- $pd.Name}}={{$pd.AdvertisePeerAddr}}
     {{- else -}}
-      ,{{- $pd.Name}}={{$pd.Scheme}}://{{$pd.IP}}:{{$pd.PeerPort}}
+      ,{{- $pd.Name}}={{$pd.AdvertisePeerAddr}}
     {{- end}}
   {{- end}}
 {{- end}}
@@ -24,11 +24,10 @@ exec bin/pd-server \
 {{- end}}
     --name="{{.Name}}" \
     --client-urls="{{.Scheme}}://{{.ListenHost}}:{{.ClientPort}}" \
-    --advertise-client-urls="{{.Scheme}}://{{.IP}}:{{.ClientPort}}" \
+    --advertise-client-urls="{{.AdvertiseClientAddr}}" \
     --peer-urls="{{.Scheme}}://{{.ListenHost}}:{{.PeerPort}}" \
-    --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \
+    --advertise-peer-urls="{{.AdvertisePeerAddr}}" \
     --data-dir="{{.DataDir}}" \
     --initial-cluster="{{template "PDList" .Endpoints}}" \
     --config=conf/pd.toml \
     --log-file="{{.LogDir}}/pd.log" 2>> "{{.LogDir}}/pd_stderr.log"
-

embed/templates/scripts/run_pd_scale.sh.tpl

@@ -10,9 +10,9 @@ cd "${DEPLOY_DIR}" || exit 1
 {{- define "PDList"}}
   {{- range $idx, $pd := .}}
     {{- if eq $idx 0}}
-      {{- $pd.Scheme}}://{{$pd.IP}}:{{$pd.ClientPort}}
+      {{- $pd.Name}}={{$pd.AdvertisePeerAddr}}
     {{- else -}}
-      ,{{- $pd.Scheme}}://{{$pd.IP}}:{{$pd.ClientPort}}
+      ,{{- $pd.Name}}={{$pd.AdvertisePeerAddr}}
     {{- end}}
   {{- end}}
 {{- end}}
@@ -24,9 +24,9 @@ exec bin/pd-server \
 {{- end}}
     --name="{{.Name}}" \
     --client-urls="{{.Scheme}}://{{.ListenHost}}:{{.ClientPort}}" \
-    --advertise-client-urls="{{.Scheme}}://{{.IP}}:{{.ClientPort}}" \
+    --advertise-client-urls="{{.AdvertiseClientAddr}}" \
     --peer-urls="{{.Scheme}}://{{.ListenHost}}:{{.PeerPort}}" \
-    --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \
+    --advertise-peer-urls="{{.AdvertisePeerAddr}}" \
     --data-dir="{{.DataDir}}" \
     --join="{{template "PDList" .Endpoints}}" \
     --config=conf/pd.toml \

embed/templates/scripts/run_tidb.sh.tpl

@@ -25,7 +25,7 @@ exec env GODEBUG=madvdontneed=1 bin/tidb-server \
     -P {{.Port}} \
     --status="{{.StatusPort}}" \
     --host="{{.ListenHost}}" \
-    --advertise-address="{{.IP}}" \
+    --advertise-address="{{.AdvertiseAddr}}" \
     --store="tikv" \
     --path="{{template "PDList" .Endpoints}}" \
     --log-slow-query="log/tidb_slow_query.log" \

embed/templates/scripts/run_tikv.sh.tpl

@@ -20,16 +20,18 @@ echo $stat
   {{- end}}
 {{- end}}
 
+export MALLOC_CONF="prof:true,prof_active:true,prof.active:false"
+
 {{- if .NumaNode}}
 exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/tikv-server \
 {{- else}}
 exec bin/tikv-server \
 {{- end}}
     --addr "{{.ListenHost}}:{{.Port}}" \
-    --advertise-addr "{{.IP}}:{{.Port}}" \
+    --advertise-addr "{{.AdvertiseAddr}}" \
     --status-addr "{{.ListenHost}}:{{.StatusPort}}" \
 {{- if .SupportAdvertiseStatusAddr}}
-    --advertise-status-addr "{{.IP}}:{{.StatusPort}}" \
+    --advertise-status-addr "{{.AdvertiseStatusAddr}}" \
 {{- end}}
     --pd "{{template "PDList" .Endpoints}}" \
     --data-dir "{{.DataDir}}" \

pkg/checkpoint/checkpoint.go

@@ -38,6 +38,9 @@ const (
 	goroutineKey = contextKey("CHECKPOINT_GOROUTINE")
 	funcKey      = "__func__"
 	hashKey      = "__hash__"
+
+	// At most 10M for each line in audit log
+	maxTokenSize = 10 * 1024 * 1024
 )
 
 var (
@@ -125,6 +128,7 @@ func NewCheckPoint(r io.Reader) (*CheckPoint, error) {
 	cp := CheckPoint{points: make([]map[string]interface{}, 0)}
 
 	scanner := bufio.NewScanner(r)
+	scanner.Buffer(nil, maxTokenSize)
 	for scanner.Scan() {
 		line := scanner.Text()
 		m, err := checkLine(line)

pkg/cluster/api/pdapi.go

@@ -30,10 +30,12 @@ import (
 	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/utils"
+	"golang.org/x/mod/semver"
 )
 
 // PDClient is an HTTP client of the PD server
 type PDClient struct {
+	version    string
 	addrs      []string
 	tlsEnabled bool
 	httpClient *utils.HTTPClient
@@ -46,11 +48,30 @@ func NewPDClient(addrs []string, timeout time.Duration, tlsConfig *tls.Config) *
 		enableTLS = true
 	}
 
-	return &PDClient{
+	cli := &PDClient{
 		addrs:      addrs,
 		tlsEnabled: enableTLS,
 		httpClient: utils.NewHTTPClient(timeout, tlsConfig),
 	}
+
+	cli.tryIdentifyVersion()
+	return cli
+}
+
+func (pc *PDClient) tryIdentifyVersion() {
+	endpoints := pc.getEndpoints(pdVersionURI)
+	response := map[string]string{}
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		body, err := pc.httpClient.Get(endpoint)
+		if err != nil {
+			return body, err
+		}
+
+		return body, json.Unmarshal(body, &response)
+	})
+	if err == nil {
+		pc.version = response["version"]
+	}
 }
 
 // GetURL builds the the client URL of PDClient
@@ -70,6 +91,7 @@ const (
 // nolint (some is unused now)
 var (
 	pdPingURI            = "pd/ping"
+	pdVersionURI         = "pd/api/v1/version"
 	pdConfigURI          = "pd/api/v1/config"
 	pdClusterIDURI       = "pd/api/v1/cluster"
 	pdConfigReplicate    = "pd/api/v1/config/replicate"
@@ -735,6 +757,11 @@ func (pc *PDClient) CheckRegion(state string) (*RegionsInfo, error) {
 // SetReplicationConfig sets a config key value of PD replication, it has the
 // same effect as `pd-ctl config set key value`
 func (pc *PDClient) SetReplicationConfig(key string, value int) error {
+	// Only support for pd version >= v4.0.0
+	if pc.version == "" || semver.Compare(pc.version, "v4.0.0") < 0 {
+		return nil
+	}
+
 	data := map[string]interface{}{"set": map[string]interface{}{key: value}}
 	body, err := json.Marshal(data)
 	if err != nil {
@@ -747,6 +774,11 @@ func (pc *PDClient) SetReplicationConfig(key string, value int) error {
 // SetAllStoreLimits sets store for all stores and types, it has the same effect
 // as `pd-ctl store limit all value`
 func (pc *PDClient) SetAllStoreLimits(value int) error {
+	// Only support for pd version >= v4.0.0
+	if pc.version == "" || semver.Compare(pc.version, "v4.0.0") < 0 {
+		return nil
+	}
+
 	data := map[string]interface{}{"rate": value}
 	body, err := json.Marshal(data)
 	if err != nil {