Merge branch 'master' into merge

CHANGELOG.md

@@ -1,5 +1,28 @@
 TiUP Changelog
 
+## [1.2.4] 2020.11.19
+
+### Fixes
+
+- Fix the issue that Pump & Drainer has different node id between tidb-ansible and TiUP ([#903](https://github.com/pingcap/tiup/pull/903), [@lucklove](https://github.com/lucklove))
+  - For the cluster imported from tidb-ansible, if the pump or drainer is restarted, it will start with a new node id
+  - Risk of this issue: binlog may not work correctly after restart pump or drainer
+- Fix the issue that audit log may get lost in some special case ([#879](https://github.com/pingcap/tiup/pull/879), [#882](https://github.com/pingcap/tiup/pull/882), [@9547](https://github.com/9547))
+  - If the user execute two commands one follows the other, and the second one quit in 1 second, the audit log of the first command will be overwirten by the second one
+  - Risk caused by this issue: some audit logs may get lost in above case
+- Fix the issue that new component deployed with `tiup cluster scale-out` doesn't auto start when rebooting ([#905](https://github.com/pingcap/tiup/pull/905), [@9547](https://github.com/9547))
+  - Risk caused by this issue: the cluster may be unavailable after rebooting
+- Fix the issue that data directory of tiflash is not deleted if multiple data directories are specified ([#871](https://github.com/pingcap/tiup/pull/871), [@9547](https://github.com/9547))
+- Fix the issue that `node_exporter` and `blackbox_exporter` not cleaned up after scale-in all instances on specified host ([#857](https://github.com/pingcap/tiup/pull/857), [@9547](https://github.com/9547))
+- Fix the issue that the patch command will fail when try to patch dm cluster ([#884](https://github.com/pingcap/tiup/pull/884), [@lucklove](https://github.com/lucklove))
+- Fix the issue that the bench component report `Error 1105: client has multi-statement capability disabled` ([#887](https://github.com/pingcap/tiup/pull/887), [@mahjonp](https://github.com/mahjonp))
+- Fix the issue that the TiSpark node can't be upgraded ([#901](https://github.com/pingcap/tiup/pull/901), [@lucklove](https://github.com/lucklove))
+- Fix the issue that tiup-playground can't start TiFlash with newest nightly PD ([#902](https://github.com/pingcap/tiup/pull/902), [@lucklove](https://github.com/lucklove))
+
+### Improvements
+
+- Ignore no tispark master error when listing clusters since the master node may be remove by `scale-in --force` ([#920](https://github.com/pingcap/tiup/pull/920), [@AstroProfundis](https://github.com/AstroProfundis))
+
 ## [1.2.3] 2020.10.30
 
 ### Fixes

cmd/env.go

@@ -30,6 +30,7 @@ var envList = []string{
 	localdata.EnvNameSCPPath,
 	localdata.EnvNameKeepSourceTarget,
 	localdata.EnvNameMirrorSyncScript,
+	localdata.EnvNameLogPath,
 }
 
 func newEnvCmd() *cobra.Command {

cmd/mirror.go

@@ -638,8 +638,8 @@ func newMirrorCloneCmd() *cobra.Command {
 			}
 			defer repo.Mirror().Close()
 
-			var versionMapper = func(ver string) string {
-				return spec.TiDBComponentVersion(ver, "")
+			var versionMapper = func(comp string) string {
+				return spec.TiDBComponentVersion(comp, "")
 			}
 
 			return repository.CloneMirror(repo, components, versionMapper, args[0], args[1:], options)

components/cluster/command/import.go

@@ -128,6 +128,11 @@ func newImportCmd() *cobra.Command {
 				return err
 			}
 
+			// copy config detail to meta file
+			if err = ansible.LoadConfig(clsName, clsMeta); err != nil {
+				return err
+			}
+
 			if err = spec.SaveClusterMeta(clsName, clsMeta); err != nil {
 				return err
 			}

components/cluster/command/root.go

@@ -298,7 +298,7 @@ func Execute() {
 		}
 
 		if !errorx.HasTrait(err, errutil.ErrTraitPreCheck) {
-			logger.OutputDebugLog()
+			logger.OutputDebugLog("tiup-cluster")
 		}
 
 		if errx := errorx.Cast(err); errx != nil {

components/cluster/command/scale_out.go

@@ -14,14 +14,11 @@
 package command
 
 import (
-	"context"
 	"io/ioutil"
 	"path/filepath"
 
 	"github.com/pingcap/tiup/pkg/cluster"
 	"github.com/pingcap/tiup/pkg/cluster/executor"
-	operator "github.com/pingcap/tiup/pkg/cluster/operation"
-	"github.com/pingcap/tiup/pkg/cluster/report"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/cluster/task"
 	"github.com/pingcap/tiup/pkg/utils"
@@ -78,15 +75,6 @@ func newScaleOutCmd() *cobra.Command {
 	return cmd
 }
 
-// Deprecated
-func convertStepDisplaysToTasks(t []*task.StepDisplay) []task.Task {
-	tasks := make([]task.Task, 0, len(t))
-	for _, sd := range t {
-		tasks = append(tasks, sd)
-	}
-	return tasks
-}
-
 func final(builder *task.Builder, name string, meta spec.Metadata) {
 	builder.UpdateTopology(name,
 		tidbSpec.Path(name),
@@ -96,15 +84,5 @@ func final(builder *task.Builder, name string, meta spec.Metadata) {
 }
 
 func postScaleOutHook(builder *task.Builder, newPart spec.Topology) {
-	nodeInfoTask := task.NewBuilder().Func("Check status", func(ctx *task.Context) error {
-		var err error
-		teleNodeInfos, err = operator.GetNodeInfo(context.Background(), ctx, newPart)
-		_ = err
-		// intend to never return error
-		return nil
-	}).BuildAsStep("Check status").SetHidden(true)
-
-	if report.Enable() {
-		builder.Parallel(false, convertStepDisplaysToTasks([]*task.StepDisplay{nodeInfoTask})...)
-	}
+	postDeployHook(builder, newPart)
 }

components/dm/ansible/import.go

@@ -222,7 +222,7 @@ func (im *Importer) handleWorkerConfig(srv *spec.WorkerSpec, fname string) error
 
 // ScpSourceToMaster scp the source files to master,
 // and set V1SourcePath of the master spec.
-func (im *Importer) ScpSourceToMaster(topo *spec.Topology) (err error) {
+func (im *Importer) ScpSourceToMaster(topo *spec.Specification) (err error) {
 	for i := 0; i < len(topo.Masters); i++ {
 		master := &topo.Masters[i]
 		target := filepath.Join(firstNonEmpty(master.DeployDir, topo.GlobalOptions.DeployDir), "v1source")
@@ -293,7 +293,7 @@ func (im *Importer) ImportFromAnsibleDir() (clusterName string, meta *spec.Metad
 	}
 
 	meta = &spec.Metadata{
-		Topology: new(spec.Topology),
+		Topology: new(spec.Specification),
 	}
 	topo := meta.Topology
 
@@ -480,7 +480,7 @@ func (im *Importer) ImportFromAnsibleDir() (clusterName string, meta *spec.Metad
 			}
 		case "alertmanager_servers":
 			for _, host := range group.Hosts {
-				srv := spec.AlertManagerSpec{
+				srv := spec.AlertmanagerSpec{
 					Host:      host.Vars["ansible_host"],
 					SSHPort:   ansible.GetHostPort(host, cfg),
 					DeployDir: firstNonEmpty(host.Vars["deploy_dir"], topo.GlobalOptions.DeployDir),
@@ -521,9 +521,9 @@ func (im *Importer) ImportFromAnsibleDir() (clusterName string, meta *spec.Metad
 					}
 				}
 
-				srv.DeployDir = instancDeployDir(spec.ComponentAlertManager, srv.WebPort, host.Vars["deploy_dir"], topo.GlobalOptions.DeployDir)
+				srv.DeployDir = instancDeployDir(spec.ComponentAlertmanager, srv.WebPort, host.Vars["deploy_dir"], topo.GlobalOptions.DeployDir)
 
-				topo.Alertmanager = append(topo.Alertmanager, srv)
+				topo.Alertmanagers = append(topo.Alertmanagers, srv)
 			}
 		case "grafana_servers":
 			for _, host := range group.Hosts {
@@ -559,7 +559,7 @@ func (im *Importer) ImportFromAnsibleDir() (clusterName string, meta *spec.Metad
 				}
 
 				srv.DeployDir = instancDeployDir(spec.ComponentGrafana, srv.Port, host.Vars["deploy_dir"], topo.GlobalOptions.DeployDir)
-				topo.Grafana = append(topo.Grafana, srv)
+				topo.Grafanas = append(topo.Grafanas, srv)
 			}
 		case "all", "ungrouped":
 			// ignore intent

components/dm/ansible/import_test.go

@@ -188,9 +188,9 @@ func TestImportFromAnsible(t *testing.T) {
 	assert.Equal(expectedWorker, worker)
 
 	// check Alertmanager
-	assert.Len(topo.Alertmanager, 1)
-	aler := topo.Alertmanager[0]
-	expectedAlter := spec.AlertManagerSpec{
+	assert.Len(topo.Alertmanagers, 1)
+	aler := topo.Alertmanagers[0]
+	expectedAlter := spec.AlertmanagerSpec{
 		Host:      "172.19.0.101",
 		SSHPort:   22,
 		WebPort:   9093,
@@ -201,8 +201,8 @@ func TestImportFromAnsible(t *testing.T) {
 	assert.Equal(expectedAlter, aler)
 
 	// Check Grafana
-	assert.Len(topo.Grafana, 1)
-	grafana := topo.Grafana[0]
+	assert.Len(topo.Grafanas, 1)
+	grafana := topo.Grafanas[0]
 	expectedGrafana := spec.GrafanaSpec{
 		Host:      "172.19.0.101",
 		SSHPort:   22,

components/dm/command/root.go

@@ -217,7 +217,7 @@ func Execute() {
 		}
 
 		if !errorx.HasTrait(err, errutil.ErrTraitPreCheck) {
-			logger.OutputDebugLog()
+			logger.OutputDebugLog("tiup-dm")
 		}
 
 		if errx := errorx.Cast(err); errx != nil {

components/dm/command/scale_in.go

@@ -75,7 +75,7 @@ func newScaleInCmd() *cobra.Command {
 // ScaleInDMCluster scale in dm cluster.
 func ScaleInDMCluster(
 	getter operator.ExecutorGetter,
-	topo *dm.Topology,
+	topo *dm.Specification,
 	options operator.Options,
 ) error {
 	// instances by uuid

components/dm/spec/bindversion.go

@@ -5,7 +5,7 @@ import "github.com/pingcap/tiup/pkg/cluster/spec"
 // DMComponentVersion maps the dm version to the third components binding version
 func DMComponentVersion(comp, version string) string {
 	switch comp {
-	case spec.ComponentAlertManager:
+	case spec.ComponentAlertmanager:
 		return "v0.17.0"
 	case spec.ComponentGrafana, spec.ComponentPrometheus:
 		return "v4.0.3"