Merge branch 'master' into fix_manifest-cache

components/playground/playground.go

@@ -1041,7 +1041,7 @@ func (p *Playground) bootMonitor(ctx context.Context, env *environment.Environme
 	}
 
 	p.instanceWaiter.Go(func() error {
-		err := p.monitor.wait()
+		err := monitor.wait()
 		if err != nil && atomic.LoadInt32(&p.curSig) == 0 {
 			fmt.Printf("Prometheus quit: %v\n", err)
 		} else {
@@ -1077,7 +1077,7 @@ func (p *Playground) bootNGMonitoring(ctx context.Context, env *environment.Envi
 	}
 
 	p.instanceWaiter.Go(func() error {
-		err := p.ngmonitoring.wait()
+		err := ngm.wait()
 		if err != nil && atomic.LoadInt32(&p.curSig) == 0 {
 			fmt.Printf("ng-monitoring quit: %v\n", err)
 		} else {
@@ -1146,7 +1146,7 @@ func (p *Playground) bootGrafana(ctx context.Context, env *environment.Environme
 	}
 
 	p.instanceWaiter.Go(func() error {
-		err := p.grafana.wait()
+		err := grafana.wait()
 		if err != nil && atomic.LoadInt32(&p.curSig) == 0 {
 			fmt.Printf("Grafana quit: %v\n", err)
 		} else {

doc/user/mirrors.md

@@ -1,55 +1,44 @@
 # Build a private mirror
 
-When building a private cloud, it is common to use an isolated network environment where the official mirror of TiUP is not accessible, so we provide a solution for building a private mirror, which is mainly implemented by the mirros component, which can also be used for offline deployment.
+When building a private cloud, it is common to use an isolated network environment where the official mirror of TiUP is not accessible, so we provide a solution for building a private mirror, which is mainly implemented by the mirror component, which can also be used for offline deployment.
 
 ## Mirrors component introduction
 
-First, let's look at the `mirrors' help file.
+First, let's look at the `mirror' help file.
 
 ```bash
-$ tiup mirrors --help
-Starting component `mirrors`: /Users/joshua/.tiup/components/mirrors/v0.0.1/mirrors
-Build a local mirrors and download all selected components
+$ tiup mirror --help
+The 'mirror' command is used to manage a component repository for TiUP, you can use
+it to create a private repository, or to add new component to an existing repository.
+The repository can be used either online or offline.
+It also provides some useful utilities to help managing keys, users and versions
+of components or the repository itself.
 
 Usage:
-  tiup mirrors <target-dir> [global-version] [flags]
-
-Examples:
-  tiup mirrors local-path --arch amd64,arm --os linux,darwin    # Specify the architectures and OSs
-  tiup mirrors local-path --full                                # Build a full local mirrors
-  tiup mirrors local-path --tikv v4                             # Specify the version via prefix
-  tiup mirrors local-path --tidb all --pd all                   # Download all version for specific component
-
-Flags:
-      --overwrite                   Overwrite the exists tarball
-  -f, --full                        Build a full mirrors repository
-  -a, --arch strings                Specify the downloading architecture (default [amd64])
-  -o, --os strings                  Specify the downloading os (default [linux,darwin])
-      --tidb strings                Specify the versions for component tidb
-      --tikv strings                Specify the versions for component tikv
-      --pd strings                  Specify the versions for component pd
-      --playground strings          Specify the versions for component playground
-      --client strings              Specify the versions for component client
-      --prometheus strings          Specify the versions for component prometheus
-      --package strings             Specify the versions for component package
-      --grafana strings             Specify the versions for component grafana
-      --alertmanager strings        Specify the versions for component alertmanager
-      --blackbox_exporter strings   Specify the versions for component blackbox_exporter
-      --node_exporter strings       Specify the versions for component node_exporter
-      --pushgateway strings         Specify the versions for component pushgateway
-      --tiflash strings             Specify the versions for component tiflash
-      --drainer strings             Specify the versions for component drainer
-      --pump strings                Specify the versions for component pump
-      --cluster strings             Specify the versions for component cluster
-      --mirrors strings             Specify the versions for component mirrors
-      --bench strings               Specify the versions for component bench
-      --insight strings             Specify the versions for component insight
-      --doc strings                 Specify the versions for component doc
-      --ctl strings                 Specify the versions for component ctl
-  -h, --help                        help for tiup
+  tiup mirror <command> [flags]
+
+Available Commands:
+  init        Initialize an empty repository
+  sign        Add signatures to a manifest file
+  genkey      Generate a new key pair
+  clone       Clone a local mirror from remote mirror and download all selected components
+  merge       Merge two or more offline mirror
+  publish     Publish a component
+  show        Show mirror address
+  set         Set mirror address
+  modify      Modify published component
+  renew       Renew the manifest of a published component.
+  grant       grant a new owner
+  rotate      Rotate root.json
+
+Global Flags:
+      --help                 Help for this command
+      --skip-version-check   Skip the strict version check, by default a version must be a valid SemVer string
+
+Use "tiup mirror [command] --help" for more information about a command.
 ```
 
-Its basic use is `tiup mirrors <target-dir> [global-version] [flags]`, the target-dir is the directory in which the cloned data needs to be placed. global-version is used to quickly set a common version for all components.
+Its basic use is `tiup mirror clone <target-dir> [global-version] [flags]`, the target-dir is the directory in which the cloned data needs to be placed. global-version is used to quickly set a common version for all components.
 
 Then this order has very scary dozens of flags and even more later. But there is no need to be intimidated by the number of these flags, which are in fact of four types.
 
@@ -68,16 +57,16 @@ If `--full` is specified, the official image will be cloned intact.
 ### 3. Platform limitation
 
 If you only want to clone packages for a particular platform, you can use `-os` and `-arch` to qualify:
-- `tiup mirros <target-dir> ---os=linux`
-- Just want to clone amd64 architecture: `tiup mirros <target-dir> --arch=amd64`
-- Just want to clone linux/amd64: `tiup mirros <target-dir> --os=linux --arch=amd64`
+- `tiup mirror clone <target-dir> ---os=linux`
+- Just want to clone amd64 architecture: `tiup mirror clone <target-dir> --arch=amd64`
+- Just want to clone linux/amd64: `tiup mirror clone <target-dir> --os=linux --arch=amd64`
 
 ### 4. Component version limited
 
 If you want to clone only one version of a component and not all versions, use `--<component>=<version>` to qualify, for example "
-- Just want to clone the v4 version of tidb: `tiup mirrors <target-dir> --tidb v4`
-- Just want to clone the v4 version of tidb, and all versions of tikv: `tiup mirros <target-dir> --tidb v4 --tikv all` 
-- Clone specific versions of all components that start a cluster: `tiup mirrors <target-dir> v4.0.0-rc`
+- Just want to clone the v4 version of tidb: `tiup mirror clone <target-dir> --tidb v4`
+- Just want to clone the v4 version of tidb, and all versions of tikv: `tiup mirror clone <target-dir> --tidb v4 --tikv all` 
+- Clone specific versions of all components that start a cluster: `tiup mirror clone <target-dir> v4.0.0-rc`
 
 ## The real thing
 
@@ -86,7 +75,7 @@ If you want to clone only one version of a component and not all versions, use `
 For example, if we want to install a v4.0.0-rc TiDB cluster in an isolated environment, we can execute the following command on a machine connected to the extranet to pull the required components:
 
 ```bash
-tiup mirrors package --os=linux v4.0.0-rc
+tiup mirror package --os=linux v4.0.0-rc
 ```
 
 This command creates a directory called `package` in the current directory that contains the package of components necessary to start a cluster, which is then packaged by the tar command and sent to a central control unit in an isolated environment:
@@ -110,7 +99,7 @@ export TIUP_MIRRORS=/path/to/mirror
 tiup cluster xxx
 ```
 
-`/path/to/mirror` is the location of <target-dir> in `tiup mirrors <target-dir>`, or if in /tmp/package:
+`/path/to/mirror` is the location of <target-dir> in `tiup mirror clone <target-dir>`, or if in /tmp/package:
 ```bash
 export TIUP_MIRRORS=/tmp/package
 ```

embed/templates/scripts/run_drainer.sh.tpl

@@ -29,5 +29,4 @@ exec bin/drainer \
     --pd-urls="{{template "PDList" .Endpoints}}" \
     --data-dir="{{.DataDir}}" \
     --log-file="{{.LogDir}}/drainer.log" \
-    --config=conf/drainer.toml \
-    --initial-commit-ts={{.CommitTs}} 2>> "{{.LogDir}}/drainer_stderr.log"
+    --config=conf/drainer.toml 2>> "{{.LogDir}}/drainer_stderr.log"

pkg/cluster/ansible/service.go

@@ -196,7 +196,8 @@ func parseDirs(ctx context.Context, user string, ins spec.InstanceSpec, sshTimeo
 			if strings.Contains(line, "--initial-commit-ts=") {
 				tsArg := strings.Split(line, " ")[4] // 4 whitespaces ahead
 				tmpTs, _ := strconv.Atoi(strings.TrimPrefix(tsArg, "--initial-commit-ts="))
-				newIns.CommitTS = int64(tmpTs)
+				newIns.Config = make(map[string]interface{})
+				newIns.Config["initial_commit_ts"] = int64(tmpTs)
 			}
 		}
 		return newIns, nil

pkg/cluster/manager/deploy.go

@@ -411,7 +411,7 @@ func (m *Manager) Deploy(
 		return err
 	}
 
-	hint := color.New(color.Bold).Sprintf("%s start %s", tui.OsArgs0(), name)
+	hint := color.New(color.Bold).Sprintf("%s start %s --init", tui.OsArgs0(), name)
 	m.logger.Infof("Cluster `%s` deployed successfully, you can start it with command: `%s`", name, hint)
 	return nil
 }

pkg/cluster/spec/drainer.go

@@ -38,7 +38,7 @@ type DrainerSpec struct {
 	DeployDir       string                 `yaml:"deploy_dir,omitempty"`
 	DataDir         string                 `yaml:"data_dir,omitempty"`
 	LogDir          string                 `yaml:"log_dir,omitempty"`
-	CommitTS        int64                  `yaml:"commit_ts" default:"-1" validate:"commit_ts:editable"`
+	CommitTS        *int64                 `yaml:"commit_ts,omitempty" validate:"commit_ts:editable"` // do not use it anymore, exist for compatibility
 	Offline         bool                   `yaml:"offline,omitempty"`
 	NumaNode        string                 `yaml:"numa_node,omitempty" validate:"numa_node:editable"`
 	Config          map[string]interface{} `yaml:"config,omitempty" validate:"config:ignore"`
@@ -168,8 +168,6 @@ func (i *DrainerInstance) InitConfig(
 		paths.Log,
 	).WithPort(spec.Port).WithNumaNode(spec.NumaNode).AppendEndpoints(topo.Endpoints(deployUser)...)
 
-	cfg.WithCommitTs(spec.CommitTS)
-
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_drainer_%s_%d.sh", i.GetHost(), i.GetPort()))
 
 	if err := cfg.ConfigToFile(fp); err != nil {

pkg/cluster/spec/spec.go

@@ -356,6 +356,9 @@ func (s *Specification) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		}
 	}
 
+	// --initial-commit-ts should not be recorded at run_drainer.sh #1682
+	s.removeCommitTS()
+
 	return s.Validate()
 }
 
@@ -866,6 +869,21 @@ func setHostArch(field reflect.Value, hostArch map[string]string) error {
 	return nil
 }
 
+// when upgrade form old tiup-cluster, replace spec.CommitTS with spec.Config["initial_commit_ts"]
+func (s *Specification) removeCommitTS() {
+	_, ok1 := s.ServerConfigs.Drainer["initial_commit_ts"]
+	for _, spec := range s.Drainers {
+		_, ok2 := spec.Config["initial_commit_ts"]
+		if !ok1 && !ok2 && spec.CommitTS != nil && *spec.CommitTS != -1 {
+			if spec.Config == nil {
+				spec.Config = make(map[string]interface{})
+			}
+			spec.Config["initial_commit_ts"] = *spec.CommitTS
+		}
+		spec.CommitTS = nil
+	}
+}
+
 // GetGrafanaConfig returns global grafana configurations
 func (s *Specification) GetGrafanaConfig() map[string]string {
 	return s.ServerConfigs.Grafana

pkg/cluster/task/builder.go

@@ -23,10 +23,12 @@ import (
 	operator "github.com/pingcap/tiup/pkg/cluster/operation"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/crypto"
+	"github.com/pingcap/tiup/pkg/environment"
 	logprinter "github.com/pingcap/tiup/pkg/logger/printer"
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/pingcap/tiup/pkg/proxy"
 	"github.com/pingcap/tiup/pkg/tui"
+	"github.com/pingcap/tiup/pkg/utils"
 )
 
 // Builder is used to build TiUP task
@@ -242,10 +244,31 @@ func (b *Builder) BackupComponent(component, fromVer string, host, deployDir str
 
 // InitConfig appends a CopyComponent task to the current task collection
 func (b *Builder) InitConfig(clusterName, clusterVersion string, specManager *spec.SpecManager, inst spec.Instance, deployUser string, ignoreCheck bool, paths meta.DirPaths) *Builder {
+	// get nightly version
+	var componentVersion utils.Version
+	meta := specManager.NewMetadata()
+
+	//  full version
+	componentVersion = utils.Version(clusterVersion)
+	if err := specManager.Metadata(clusterName, meta); err == nil {
+		// get nightly version
+		if clusterVersion == utils.NightlyVersionAlias {
+			componentVersion, _, err = environment.GlobalEnv().V1Repository().LatestNightlyVersion(inst.ComponentName())
+			if err != nil {
+				componentVersion = utils.Version(clusterVersion)
+			}
+		}
+
+		// dm cluster does not require a full nightly version
+		if meta.GetTopology().Type() == spec.TopoTypeDM {
+			componentVersion = utils.Version(clusterVersion)
+		}
+	}
+
 	b.tasks = append(b.tasks, &InitConfig{
 		specManager:    specManager,
 		clusterName:    clusterName,
-		clusterVersion: clusterVersion,
+		clusterVersion: string(componentVersion),
 		instance:       inst,
 		deployUser:     deployUser,
 		ignoreCheck:    ignoreCheck,

pkg/cluster/template/scripts/drainer.go

@@ -31,7 +31,6 @@ type DrainerScript struct {
 	DataDir   string
 	LogDir    string
 	NumaNode  string
-	CommitTs  int64
 	Endpoints []*PDScript
 }
 
@@ -44,7 +43,6 @@ func NewDrainerScript(nodeID, ip, deployDir, dataDir, logDir string) *DrainerScr
 		DeployDir: deployDir,
 		DataDir:   dataDir,
 		LogDir:    logDir,
-		CommitTs:  -1,
 	}
 }
 
@@ -60,12 +58,6 @@ func (c *DrainerScript) WithNumaNode(numa string) *DrainerScript {
 	return c
 }
 
-// WithCommitTs set CommitTs field of DrainerScript
-func (c *DrainerScript) WithCommitTs(ts int64) *DrainerScript {
-	c.CommitTs = ts
-	return c
-}
-
 // AppendEndpoints add new DrainerScript to Endpoints field
 func (c *DrainerScript) AppendEndpoints(ends ...*PDScript) *DrainerScript {
 	c.Endpoints = append(c.Endpoints, ends...)

pkg/repository/clone_mirror.go

@@ -82,15 +82,15 @@ func CloneMirror(repo *V1Repository,
 	}
 
 	var (
-		initTime = time.Now()
-		expirsAt = initTime.Add(50 * 365 * 24 * time.Hour)
-		root     = v1manifest.NewRoot(initTime)
-		index    = v1manifest.NewIndex(initTime)
+		initTime  = time.Now()
+		expiresAt = initTime.Add(50 * 365 * 24 * time.Hour)
+		root      = v1manifest.NewRoot(initTime)
+		index     = v1manifest.NewIndex(initTime)
 	)
 
 	// All offline expires at 50 years to prevent manifests stale
-	root.SetExpiresAt(expirsAt)
-	index.SetExpiresAt(expirsAt)
+	root.SetExpiresAt(expiresAt)
+	index.SetExpiresAt(expiresAt)
 
 	keys := map[string][]*v1manifest.KeyInfo{}
 	for _, ty := range []string{
@@ -148,15 +148,15 @@ func CloneMirror(repo *V1Repository,
 	}
 
 	snapshot := v1manifest.NewSnapshot(initTime)
-	snapshot.SetExpiresAt(expirsAt)
+	snapshot.SetExpiresAt(expiresAt)
 
 	componentManifests, err := cloneComponents(repo, components, selectedVersions, tidbClusterVersionMapper, targetDir, tmpDir, options)
 	if err != nil {
 		return err
 	}
 
 	for name, component := range componentManifests {
-		component.SetExpiresAt(expirsAt)
+		component.SetExpiresAt(expiresAt)
 		fname := fmt.Sprintf("%s.json", name)
 		// TODO: support external owner
 		signedManifests[component.ID], err = v1manifest.SignManifest(component, ownerkeyInfo)
@@ -179,7 +179,7 @@ func CloneMirror(repo *V1Repository,
 
 	// Initialize timestamp
 	timestamp := v1manifest.NewTimestamp(initTime)
-	timestamp.SetExpiresAt(expirsAt)
+	timestamp.SetExpiresAt(expiresAt)
 
 	manifests[v1manifest.ManifestTypeTimestamp] = timestamp
 	manifests[v1manifest.ManifestTypeSnapshot] = snapshot