cluster: implement downloading of tispark packages

components/cluster/command/deploy.go

@@ -367,6 +367,11 @@ func buildMonitoredDeployTask(
 		version := spec.ComponentVersion(comp, version)
 
 		for host, info := range uniqueHosts {
+			// FIXME: as the uniqueHosts list is built with os-arch as part of the key,
+			// for platform independent packages, it will be downloaded multiple times
+			// and be saved with different file names in the packages dir, the tarballs
+			// are identical and only the difference is platform in filename.
+
 			// populate unique os/arch set
 			key := fmt.Sprintf("%s-%s-%s", comp, info.os, info.arch)
 			if _, found := uniqueCompOSArch[key]; !found {

pkg/cliutil/prepare/prepare.go

@@ -288,6 +288,11 @@ func BuildDownloadCompTasks(version string, topo *spec.Specification) []*task.St
 		if _, found := uniqueTaskList[key]; !found {
 			uniqueTaskList[key] = struct{}{}
 
+			// download spark as dependency of tispark
+			if inst.ComponentName() == spec.ComponentTiSpark {
+				tasks = append(tasks, buildDownloadSparkTask(version, inst))
+			}
+
 			version := spec.ComponentVersion(inst.ComponentName(), version)
 			t := task.NewBuilder().
 				Download(inst.ComponentName(), inst.OS(), inst.Arch(), version).
@@ -298,3 +303,13 @@ func BuildDownloadCompTasks(version string, topo *spec.Specification) []*task.St
 	})
 	return tasks
 }
+
+// buildDownloadSparkTask build download task for spark, which is a dependency of tispark
+// FIXME: this is a hack and should be replaced by dependency handling in manifest processing
+func buildDownloadSparkTask(version string, inst spec.Instance) *task.StepDisplay {
+	ver := spec.ComponentVersion(spec.ComponentSpark, version)
+	return task.NewBuilder().
+		Download(spec.ComponentSpark, inst.OS(), inst.Arch(), ver).
+		BuildAsStep(fmt.Sprintf("  - Download %s:%s (%s/%s)",
+			spec.ComponentSpark, version, inst.OS(), inst.Arch()))
+}

pkg/cluster/spec/bindversion.go

@@ -26,7 +26,7 @@ func ComponentVersion(comp, version string) string {
 		return "v0.7.0"
 	case ComponentCheckCollector:
 		return "v0.3.1"
-	case ComponentTiSparkMaster, ComponentTiSparkSlave:
+	case ComponentTiSpark:
 		return "v2.3.0-rc.1"
 	case ComponentSpark:
 		return "v2.4.3"

pkg/cluster/spec/instance.go

@@ -40,16 +40,14 @@ const (
 	ComponentDrainer          = "drainer"
 	ComponentPump             = "pump"
 	ComponentCDC              = "cdc"
-	ComponentTiSparkMaster    = "tispark-master"
-	ComponentTiSparkSlave     = "tispark-slave"
+	ComponentTiSpark          = "tispark"
 	ComponentSpark            = "spark"
 	ComponentAlertManager     = "alertmanager"
 	ComponentPrometheus       = "prometheus"
 	ComponentPushwaygate      = "pushgateway"
 	ComponentBlackboxExporter = "blackbox_exporter"
 	ComponentNodeExporter     = "node_exporter"
 	ComponentCheckCollector   = "insight"
-	platformIndependentName   = "any"
 )
 
 // Component represents a component of the cluster.
@@ -278,15 +276,15 @@ func (i *instance) LogDir() string {
 func (i *instance) OS() string {
 	v := reflect.ValueOf(i.InstanceSpec).FieldByName("OS")
 	if !v.IsValid() {
-		return platformIndependentName
+		return ""
 	}
 	return v.Interface().(string)
 }
 
 func (i *instance) Arch() string {
 	v := reflect.ValueOf(i.InstanceSpec).FieldByName("Arch")
 	if !v.IsValid() {
-		return platformIndependentName
+		return ""
 	}
 	return v.Interface().(string)
 }

pkg/cluster/spec/tispark.go

@@ -26,11 +26,13 @@ type TiSparkMasterSpec struct {
 	Port      int    `yaml:"port" default:"7077"`
 	WebPort   int    `yaml:"web_port" default:"8080"`
 	DeployDir string `yaml:"deploy_dir,omitempty"`
+	Arch      string `yaml:"arch,omitempty"`
+	OS        string `yaml:"os,omitempty"`
 }
 
 // Role returns the component role of the instance
 func (s TiSparkMasterSpec) Role() string {
-	return ComponentTiSparkMaster
+	return ComponentTiSpark
 }
 
 // SSH returns the host and SSH port of the instance
@@ -56,11 +58,13 @@ type TiSparkSlaveSpec struct {
 	Port      int    `yaml:"port" default:"7078"`
 	WebPort   int    `yaml:"web_port" default:"8081"`
 	DeployDir string `yaml:"deploy_dir,omitempty"`
+	Arch      string `yaml:"arch,omitempty"`
+	OS        string `yaml:"os,omitempty"`
 }
 
 // Role returns the component role of the instance
 func (s TiSparkSlaveSpec) Role() string {
-	return ComponentTiSparkSlave
+	return ComponentTiSpark
 }
 
 // SSH returns the host and SSH port of the instance
@@ -83,7 +87,7 @@ type TiSparkMasterComponent struct{ *Specification }
 
 // Name implements Component interface.
 func (c *TiSparkMasterComponent) Name() string {
-	return ComponentTiSparkMaster
+	return ComponentTiSpark
 }
 
 // Instances implements Component interface.
@@ -192,7 +196,7 @@ type TiSparkSlaveComponent struct{ *Specification }
 
 // Name implements Component interface.
 func (c *TiSparkSlaveComponent) Name() string {
-	return ComponentTiSparkSlave
+	return ComponentTiSpark
 }
 
 // Instances implements Component interface.