More safe way to cleanup tombstone nodes

components/cluster/command/display.go

@@ -23,11 +23,8 @@ import (
 	"github.com/fatih/color"
 	perrs "github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/cluster/api"
-	operator "github.com/pingcap/tiup/pkg/cluster/operation"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
-	"github.com/pingcap/tiup/pkg/cluster/task"
 	"github.com/pingcap/tiup/pkg/crypto"
-	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/spf13/cobra"
 )
@@ -70,12 +67,7 @@ func newDisplayCmd() *cobra.Command {
 				return displayDashboardInfo(clusterName, tlsCfg)
 			}
 
-			err = manager.Display(clusterName, gOpt)
-			if err != nil {
-				return perrs.AddStack(err)
-			}
-
-			return destroyTombstoneIfNeed(clusterName, metadata, gOpt)
+			return manager.Display(clusterName, gOpt)
 		},
 	}
 
@@ -133,48 +125,3 @@ func displayDashboardInfo(clusterName string, tlsCfg *tls.Config) error {
 
 	return nil
 }
-
-func destroyTombstoneIfNeed(clusterName string, metadata *spec.ClusterMeta, opt operator.Options) error {
-	topo := metadata.Topology
-
-	if !operator.NeedCheckTomebsome(topo) {
-		return nil
-	}
-
-	tlsCfg, err := topo.TLSConfig(tidbSpec.Path(clusterName, spec.TLSCertKeyDir))
-	if err != nil {
-		return perrs.AddStack(err)
-	}
-
-	ctx := task.NewContext()
-	err = ctx.SetSSHKeySet(spec.ClusterPath(clusterName, "ssh", "id_rsa"),
-		spec.ClusterPath(clusterName, "ssh", "id_rsa.pub"))
-	if err != nil {
-		return perrs.AddStack(err)
-	}
-
-	err = ctx.SetClusterSSH(topo, metadata.User, gOpt.SSHTimeout, gOpt.SSHType, topo.BaseTopo().GlobalOptions.SSHType)
-	if err != nil {
-		return perrs.AddStack(err)
-	}
-
-	nodes, err := operator.DestroyTombstone(ctx, topo, true /* returnNodesOnly */, opt, tlsCfg)
-	if err != nil {
-		return perrs.AddStack(err)
-	}
-
-	if len(nodes) == 0 {
-		return nil
-	}
-
-	log.Infof("Start destroy Tombstone nodes: %v ...", nodes)
-
-	_, err = operator.DestroyTombstone(ctx, topo, false /* returnNodesOnly */, opt, tlsCfg)
-	if err != nil {
-		return perrs.AddStack(err)
-	}
-
-	log.Infof("Destroy success")
-
-	return spec.SaveClusterMeta(clusterName, metadata)
-}

components/cluster/command/prune.go

@@ -0,0 +1,102 @@
+// Copyright 2020 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	perrs "github.com/pingcap/errors"
+	"github.com/pingcap/tiup/pkg/cliutil"
+	operator "github.com/pingcap/tiup/pkg/cluster/operation"
+	"github.com/pingcap/tiup/pkg/cluster/spec"
+	"github.com/pingcap/tiup/pkg/cluster/task"
+	"github.com/pingcap/tiup/pkg/logger/log"
+	"github.com/spf13/cobra"
+)
+
+func newPruneCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "prune <cluster-name>",
+		Short: "Destroy and remove instances that is in tombstone state",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return cmd.Help()
+			}
+
+			clusterName := args[0]
+
+			metadata, err := spec.ClusterMetadata(clusterName)
+			if err != nil {
+				return err
+			}
+
+			return destroyTombstoneIfNeed(clusterName, metadata, gOpt, skipConfirm)
+		},
+	}
+
+	return cmd
+}
+
+func destroyTombstoneIfNeed(clusterName string, metadata *spec.ClusterMeta, opt operator.Options, skipConfirm bool) error {
+	topo := metadata.Topology
+
+	if !operator.NeedCheckTomebsome(topo) {
+		return nil
+	}
+
+	tlsCfg, err := topo.TLSConfig(tidbSpec.Path(clusterName, spec.TLSCertKeyDir))
+	if err != nil {
+		return perrs.AddStack(err)
+	}
+
+	ctx := task.NewContext()
+	err = ctx.SetSSHKeySet(spec.ClusterPath(clusterName, "ssh", "id_rsa"),
+		spec.ClusterPath(clusterName, "ssh", "id_rsa.pub"))
+	if err != nil {
+		return perrs.AddStack(err)
+	}
+
+	err = ctx.SetClusterSSH(topo, metadata.User, gOpt.SSHTimeout, gOpt.SSHType, topo.BaseTopo().GlobalOptions.SSHType)
+	if err != nil {
+		return perrs.AddStack(err)
+	}
+
+	nodes, err := operator.DestroyTombstone(ctx, topo, true /* returnNodesOnly */, opt, tlsCfg)
+	if err != nil {
+		return perrs.AddStack(err)
+	}
+
+	if len(nodes) == 0 {
+		return nil
+	}
+
+	err = cliutil.PromptForConfirmOrAbortError(
+		color.HiYellowString(fmt.Sprintf("Will destroy these nodes: %v\nDo you confirm this action? [y/N]:", nodes)),
+	)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Start destroy Tombstone nodes: %v ...", nodes)
+
+	_, err = operator.DestroyTombstone(ctx, topo, false /* returnNodesOnly */, opt, tlsCfg)
+	if err != nil {
+		return perrs.AddStack(err)
+	}
+
+	log.Infof("Destroy success")
+
+	return spec.SaveClusterMeta(clusterName, metadata)
+}

components/cluster/command/root.go

@@ -153,6 +153,7 @@ func init() {
 		newUpgradeCmd(),
 		newExecCmd(),
 		newDisplayCmd(),
+		newPruneCmd(),
 		newListCmd(),
 		newAuditCmd(),
 		newImportCmd(),

go.sum

@@ -639,6 +639,7 @@ github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J
 github.com/pingcap/parser v0.0.0-20200422082501-7329d80eaf2c/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4=
 github.com/pingcap/pd v2.1.5+incompatible h1:vOLV2tSQdRjjmxaTXtJULoC94dYQOd+6fzn2yChODHc=
 github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
+github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2 h1:JTzYYukREvxVSKW/ncrzNjFitd8snoQ/Xz32pw8i+s8=
 github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2/go.mod h1:s+utZtXDznOiL24VK0qGmtoHjjXNsscJx3m1n8cC56s=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200408114249-ed3bd6f7fdb1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=

pkg/cluster/manager.go

@@ -599,14 +599,20 @@ func (m *Manager) Display(clusterName string, opt operator.Options) error {
 	cliutil.PrintTable(clusterTable, true)
 	fmt.Printf("Total nodes: %d\n", len(clusterTable)-1)
 
-	if _, ok := topo.(*spec.Specification); ok {
+	if t, ok := topo.(*spec.Specification); ok {
 		// Check if TiKV's label set correctly
 		pdClient := api.NewPDClient(pdList, 10*time.Second, tlsCfg)
 		if lbs, err := pdClient.GetLocationLabels(); err != nil {
 			color.Yellow("\nWARN: get location labels from pd failed: %v", err)
 		} else if err := spec.CheckTiKVLocationLabels(lbs, pdClient); err != nil {
 			color.Yellow("\nWARN: there is something wrong with TiKV labels, which may cause data losing:\n%v", err)
 		}
+
+		// Check if there is some instance in tombstone state
+		nodes, _ := operator.DestroyTombstone(ctx, t, true /* returnNodesOnly */, opt, tlsCfg)
+		if len(nodes) != 0 {
+			color.Green("There is some nodes in state Tombstone, you can destroy them with the command `tiup cluster prune %s`", clusterName)
+		}
 	}
 
 	return nil

pkg/cluster/operation/action.go

@@ -136,27 +136,28 @@ func Stop(
 }
 
 // NeedCheckTomebsome return true if we need to check and destroy some node.
-func NeedCheckTomebsome(spec *spec.Specification) bool {
-	for _, s := range spec.TiKVServers {
+func NeedCheckTomebsome(topo *spec.Specification) bool {
+	for _, s := range topo.TiKVServers {
 		if s.Offline {
 			return true
 		}
 	}
-	for _, s := range spec.TiFlashServers {
+	for _, s := range topo.TiFlashServers {
 		if s.Offline {
 			return true
 		}
 	}
-	for _, s := range spec.PumpServers {
+	for _, s := range topo.PumpServers {
 		if s.Offline {
 			return true
 		}
 	}
-	for _, s := range spec.Drainers {
+	for _, s := range topo.Drainers {
 		if s.Offline {
 			return true
 		}
 	}
+
 	return false
 }
 

pkg/cluster/operation/scale_in.go

@@ -237,7 +237,7 @@ func ScaleInCluster(
 					return errors.Annotatef(err, "failed to destroy %s", component.Name())
 				}
 			} else {
-				log.Warnf(color.YellowString("The component `%s` will be destroyed when display cluster info when it become tombstone, maybe exists in several minutes or hours",
+				log.Warnf(color.YellowString("The component `%s` will become tombstone, maybe exists in several minutes or hours, after that you can use the prune command to clean it",
 					component.Name()))
 			}
 		}

tests/tiup-cluster/script/util.sh

@@ -16,6 +16,7 @@ function instance_num() {
         client="--native-ssh"
     fi
 
+    tiup-cluster $client prune $name --yes
     count=$(tiup-cluster $client display $name | grep "Total nodes" | awk -F ' ' '{print $3}')
 
     echo $count