cluster: start pd,dm-master in sequentially (#1262)

pingcap · Apr 2, 2021 · cf55cab · cf55cab
1 parent 7e1f73b
commit cf55cab
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 1 deletion.
diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go
@@ -306,7 +306,7 @@ func buildScaleOutTask(
 			return m.specManager.SaveMeta(name, metadata)
 		}).
 		Func("StartCluster", func(ctx context.Context) error {
-			return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout}, tlsCfg)
+			return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg)
 		}).
 		Parallel(false, refreshConfigTasks...).
 		Parallel(false, buildReloadPromTasks(metadata.GetTopology())...)

diff --git a/pkg/cluster/operation/action.go b/pkg/cluster/operation/action.go
@@ -460,6 +460,14 @@ func StartComponent(ctx context.Context, instances []spec.Instance, options Opti
 	name := instances[0].ComponentName()
 	log.Infof("Starting component %s", name)
 
+	// start instances in serial for Raft related components
+	// eg: PD has more strict restrictions on the capacity expansion process,
+	// that is, there should be only one node in the peer-join stage at most
+	// ref https://github.com/tikv/pd/blob/d38b36714ccee70480c39e07126e3456b5fb292d/server/join/join.go#L179-L191
+	if options.Operation == ScaleOutOperation && (name == spec.ComponentPD || name == spec.ComponentDMMaster) {
+		return serialStartInstances(ctx, instances, options, tlsCfg)
+	}
+
 	errg, _ := errgroup.WithContext(ctx)
 
 	for _, ins := range instances {
@@ -484,6 +492,18 @@ func StartComponent(ctx context.Context, instances []spec.Instance, options Opti
 	return errg.Wait()
 }
 
+func serialStartInstances(ctx context.Context, instances []spec.Instance, options Options, tlsCfg *tls.Config) error {
+	for _, ins := range instances {
+		if err := ins.PrepareStart(ctx, tlsCfg); err != nil {
+			return err
+		}
+		if err := startInstance(ctx, ins, options.OptTimeout); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // StopMonitored stop BlackboxExporter and NodeExporter
 func StopMonitored(ctx context.Context, instance spec.Instance, options *spec.MonitoredOptions, timeout uint64) error {
 	ports := map[string]int{

diff --git a/pkg/cluster/operation/operation.go b/pkg/cluster/operation/operation.go
@@ -43,6 +43,7 @@ type Options struct {
 
 	// Show uptime or not
 	ShowUptime bool
+	Operation  Operation
 }
 
 // Operation represents the type of cluster operation