From 4ffe6e5178c1a3307290c778226f83b1a46a0b17 Mon Sep 17 00:00:00 2001 From: unbyte Date: Thu, 22 Apr 2021 15:59:53 +0800 Subject: [PATCH] playground: support semver constraint (#1325) --- cmd/mirror.go | 2 +- components/playground/playground.go | 92 ++++++------- pkg/repository/v1_repository.go | 57 +++++++- pkg/utils/regexp.go | 14 ++ pkg/utils/regexp_test.go | 34 +++++ pkg/utils/semver.go | 195 ++++++++++++++++++++++++++++ pkg/utils/semver_test.go | 50 +++++++ pkg/utils/utils.go | 6 + 8 files changed, 402 insertions(+), 48 deletions(-) create mode 100644 pkg/utils/regexp.go create mode 100644 pkg/utils/regexp_test.go diff --git a/cmd/mirror.go b/cmd/mirror.go index 52cea62c34..fc6e772dde 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -341,7 +341,7 @@ func newMirrorRotateCmd() *cobra.Command { } func editLatestRootManifest() (*v1manifest.Root, error) { - root, err := environment.GlobalEnv().V1Repository().FetchRootManfiest() + root, err := environment.GlobalEnv().V1Repository().FetchRootManifest() if err != nil { return nil, err } diff --git a/components/playground/playground.go b/components/playground/playground.go index 3071df7af7..81371cc1c4 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -37,6 +37,7 @@ import ( "github.com/pingcap/tiup/components/playground/instance" "github.com/pingcap/tiup/pkg/cliutil/progress" "github.com/pingcap/tiup/pkg/cluster/api" + "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/utils" "golang.org/x/mod/semver" @@ -264,7 +265,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { } switch cid { - case "pd": + case spec.ComponentPD: for i := 0; i < len(p.pds); i++ { if p.pds[i].Pid() == pid { inst := p.pds[i] @@ -275,7 +276,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { p.pds = append(p.pds[:i], p.pds[i+1:]...) } } - case "tikv": + case spec.ComponentTiKV: for i := 0; i < len(p.tikvs); i++ { if p.tikvs[i].Pid() == pid { inst := p.tikvs[i] @@ -289,19 +290,19 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { return nil } } - case "tidb": + case spec.ComponentTiDB: for i := 0; i < len(p.tidbs); i++ { if p.tidbs[i].Pid() == pid { p.tidbs = append(p.tidbs[:i], p.tidbs[i+1:]...) } } - case "ticdc": + case spec.ComponentCDC: for i := 0; i < len(p.ticdcs); i++ { if p.ticdcs[i].Pid() == pid { p.ticdcs = append(p.ticdcs[:i], p.ticdcs[i+1:]...) } } - case "tiflash": + case spec.ComponentTiFlash: for i := 0; i < len(p.tiflashs); i++ { if p.tiflashs[i].Pid() == pid { inst := p.tiflashs[i] @@ -315,7 +316,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { return nil } } - case "pump": + case spec.ComponentPump: for i := 0; i < len(p.pumps); i++ { if p.pumps[i].Pid() == pid { inst := p.pumps[i] @@ -334,7 +335,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { return nil } } - case "drainer": + case spec.ComponentDrainer: for i := 0; i < len(p.drainers); i++ { if p.drainers[i].Pid() == pid { inst := p.drainers[i] @@ -391,19 +392,19 @@ func (p *Playground) sanitizeConfig(boot instance.Config, cfg *instance.Config) func (p *Playground) sanitizeComponentConfig(cid string, cfg *instance.Config) error { switch cid { - case "pd": + case spec.ComponentPD: return p.sanitizeConfig(p.bootOptions.pd, cfg) - case "tikv": + case spec.ComponentTiKV: return p.sanitizeConfig(p.bootOptions.tikv, cfg) - case "tidb": + case spec.ComponentTiDB: return p.sanitizeConfig(p.bootOptions.tidb, cfg) - case "tiflash": + case spec.ComponentTiFlash: return p.sanitizeConfig(p.bootOptions.tiflash, cfg) - case "ticdc": + case spec.ComponentCDC: return p.sanitizeConfig(p.bootOptions.ticdc, cfg) - case "pump": + case spec.ComponentPump: return p.sanitizeConfig(p.bootOptions.pump, cfg) - case "drainer": + case spec.ComponentDrainer: return p.sanitizeConfig(p.bootOptions.drainer, cfg) default: return fmt.Errorf("unknown %s in sanitizeConfig", cid) @@ -530,48 +531,48 @@ func (p *Playground) RWalkInstances(fn func(componentID string, ins instance.Ins // WalkInstances call fn for every instance and stop if return not nil. func (p *Playground) WalkInstances(fn func(componentID string, ins instance.Instance) error) error { for _, ins := range p.pds { - err := fn("pd", ins) + err := fn(spec.ComponentPD, ins) if err != nil { return err } } for _, ins := range p.tikvs { - err := fn("tikv", ins) + err := fn(spec.ComponentTiKV, ins) if err != nil { return err } } for _, ins := range p.pumps { - err := fn("pump", ins) + err := fn(spec.ComponentPump, ins) if err != nil { return err } } for _, ins := range p.tidbs { - err := fn("tidb", ins) + err := fn(spec.ComponentTiDB, ins) if err != nil { return err } } for _, ins := range p.ticdcs { - err := fn("ticdc", ins) + err := fn(spec.ComponentCDC, ins) if err != nil { return err } } for _, ins := range p.drainers { - err := fn("drainer", ins) + err := fn(spec.ComponentDrainer, ins) if err != nil { return err } } for _, ins := range p.tiflashs { - err := fn("tiflash", ins) + err := fn(spec.ComponentTiFlash, ins) if err != nil { return err } @@ -612,7 +613,7 @@ func (p *Playground) addInstance(componentID string, cfg instance.Config) (ins i host = instance.AdvertiseHost(host) switch componentID { - case "pd": + case spec.ComponentPD: inst := instance.NewPDInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id) ins = inst if p.booted { @@ -624,27 +625,27 @@ func (p *Playground) addInstance(componentID string, cfg instance.Config) (ins i pd.InitCluster(p.pds) } } - case "tidb": + case spec.ComponentTiDB: inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, p.enableBinlog()) ins = inst p.tidbs = append(p.tidbs, inst) - case "tikv": + case spec.ComponentTiKV: inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) ins = inst p.tikvs = append(p.tikvs, inst) - case "tiflash": + case spec.ComponentTiFlash: inst := instance.NewTiFlashInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, p.tidbs) ins = inst p.tiflashs = append(p.tiflashs, inst) - case "ticdc": + case spec.ComponentCDC: inst := instance.NewTiCDC(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) ins = inst p.ticdcs = append(p.ticdcs, inst) - case "pump": + case spec.ComponentPump: inst := instance.NewPump(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) ins = inst p.pumps = append(p.pumps, inst) - case "drainer": + case spec.ComponentDrainer: inst := instance.NewDrainer(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) ins = inst p.drainers = append(p.drainers, inst) @@ -670,19 +671,20 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme return fmt.Errorf("all components count must be great than 0 (tikv=%v, pd=%v)", options.tikv.Num, options.pd.Num) } - if options.version == "" { - version, _, err := env.V1Repository().LatestStableVersion("tidb", false) + { + version, err := env.V1Repository().ResolveComponentVersion(spec.ComponentTiDB, options.version) if err != nil { return err } - options.version = version.String() - - fmt.Println(color.YellowString(`No TiDB version specified, using the latest stable version: %s + fmt.Println(color.YellowString(`Using the version %s for version constraint "%s". If you'd like to use a TiDB version other than %s, cancel and retry with the following arguments: Specify version manually: tiup playground + Specify version range: tiup playground ^5 The nightly version: tiup playground nightly -`, options.version, options.version)) +`, version, options.version, version)) + + options.version = version.String() } if !utils.Version(options.version).IsNightly() { @@ -700,13 +702,13 @@ If you'd like to use a TiDB version other than %s, cancel and retry with the fol comp string instance.Config }{ - {"pd", options.pd}, - {"tikv", options.tikv}, - {"pump", options.pump}, - {"tidb", options.tidb}, - {"ticdc", options.ticdc}, - {"drainer", options.drainer}, - {"tiflash", options.tiflash}, + {spec.ComponentPD, options.pd}, + {spec.ComponentTiKV, options.tikv}, + {spec.ComponentPump, options.pump}, + {spec.ComponentTiDB, options.tidb}, + {spec.ComponentCDC, options.ticdc}, + {spec.ComponentDrainer, options.drainer}, + {spec.ComponentTiFlash, options.tiflash}, } for _, inst := range instances { @@ -758,7 +760,7 @@ If you'd like to use a TiDB version other than %s, cancel and retry with the fol anyPumpReady := false // Start all instance except tiflash. err := p.WalkInstances(func(cid string, ins instance.Instance) error { - if cid == "tiflash" { + if cid == spec.ComponentTiFlash { return nil } @@ -768,7 +770,7 @@ If you'd like to use a TiDB version other than %s, cancel and retry with the fol } // if no any pump, tidb will quit right away. - if cid == "pump" && !anyPumpReady { + if cid == spec.ComponentPump && !anyPumpReady { ctx, cancel := context.WithTimeout(context.TODO(), time.Second*120) err = ins.(*instance.Pump).Ready(ctx) cancel() @@ -883,7 +885,7 @@ If you'd like to use a TiDB version other than %s, cancel and retry with the fol } if monitorInfo != nil { - p.updateMonitorTopology("prometheus", *monitorInfo) + p.updateMonitorTopology(spec.ComponentPrometheus, *monitorInfo) } dumpDSN(filepath.Join(p.dataDir, "dsn"), p.tidbs) @@ -899,7 +901,7 @@ If you'd like to use a TiDB version other than %s, cancel and retry with the fol logIfErr(p.renderSDFile()) if g := p.grafana; g != nil { - p.updateMonitorTopology("grafana", MonitorInfo{g.host, g.port, g.cmd.Path}) + p.updateMonitorTopology(spec.ComponentGrafana, MonitorInfo{g.host, g.port, g.cmd.Path}) } return nil diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index e7117c601f..d422b3aeb4 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -23,6 +23,7 @@ import ( "path" "path/filepath" "runtime" + "sort" "strconv" "strings" "time" @@ -635,8 +636,8 @@ func (r *V1Repository) loadRoot() (*v1manifest.Root, error) { return root, nil } -// FetchRootManfiest fetch the root manifest. -func (r *V1Repository) FetchRootManfiest() (root *v1manifest.Root, err error) { +// FetchRootManifest fetch the root manifest. +func (r *V1Repository) FetchRootManifest() (root *v1manifest.Root, err error) { err = r.ensureManifests() if err != nil { return nil, err @@ -740,6 +741,58 @@ func (r *V1Repository) ComponentVersion(id, ver string, includeYanked bool) (*v1 return vi, nil } +// ResolveComponentVersion resolves the latest version of a component that satisfies the constraint +func (r *V1Repository) ResolveComponentVersion(id, constraint string) (utils.Version, error) { + manifest, err := r.FetchComponentManifest(id, false) + if err != nil { + return "", err + } + var ver string + switch constraint { + case "", utils.LatestVersionAlias: + v, _, err := r.LatestStableVersion(id, false) + if err != nil { + return "", err + } + ver = v.String() + case utils.NightlyVersionAlias: + if !manifest.HasNightly(r.PlatformString()) { + return "", errors.Annotatef(ErrUnknownVersion, "component %s does not have nightly on %s", id, r.PlatformString()) + } + ver = manifest.Nightly + default: + cons, err := utils.NewConstraint(constraint) + if err != nil { + return "", err + } + versions := manifest.VersionList(r.PlatformString()) + verList := make([]string, 0, len(versions)) + for v := range versions { + if v == manifest.Nightly { + continue + } + verList = append(verList, v) + } + sort.Slice(verList, func(p, q int) bool { + return semver.Compare(verList[p], verList[q]) > 0 + }) + for _, v := range verList { + if cons.Check(v) { + ver = v + break + } + } + } + if ver == "" { + return "", fmt.Errorf(`no version on %s for component %s satisfies constraint "%s"`, r.PlatformString(), id, constraint) + } + vi := manifest.VersionItem(r.PlatformString(), ver, false) + if vi == nil { + return "", errors.Annotatef(ErrUnknownVersion, "version %s on %s for component %s not found", ver, r.PlatformString(), id) + } + return utils.Version(ver), nil +} + // LatestNightlyVersion returns the latest nightly version of specific component func (r *V1Repository) LatestNightlyVersion(id string) (utils.Version, *v1manifest.VersionItem, error) { com, err := r.FetchComponentManifest(id, false) diff --git a/pkg/utils/regexp.go b/pkg/utils/regexp.go new file mode 100644 index 0000000000..947c5fd111 --- /dev/null +++ b/pkg/utils/regexp.go @@ -0,0 +1,14 @@ +package utils + +import "regexp" + +// MatchGroups turns a slice of matched string to a map according to capture group name +func MatchGroups(r *regexp.Regexp, str string) map[string]string { + matched := r.FindStringSubmatch(str) + results := make(map[string]string) + names := r.SubexpNames() + for i, value := range matched { + results[names[i]] = value + } + return results +} diff --git a/pkg/utils/regexp_test.go b/pkg/utils/regexp_test.go new file mode 100644 index 0000000000..dde2d860eb --- /dev/null +++ b/pkg/utils/regexp_test.go @@ -0,0 +1,34 @@ +package utils + +import ( + . "github.com/pingcap/check" + "regexp" +) + +var _ = Suite(&TestRegexpSuite{}) + +type TestRegexpSuite struct{} + +func (s *TestRegexpSuite) TestMatchGroups(c *C) { + cases := []struct { + re string + str string + expected map[string]string + }{ + { + re: `^(?P[a-zA-Z]*)(?P[0-9]*)$`, + str: "abc123", + expected: map[string]string{ + "": "abc123", + "first": "abc", + "second": "123", + }, + }, + } + + for _, cas := range cases { + c.Assert( + MatchGroups(regexp.MustCompile(cas.re), cas.str), + DeepEquals, cas.expected) + } +} diff --git a/pkg/utils/semver.go b/pkg/utils/semver.go index 7f51ee0761..b237ba8b57 100644 --- a/pkg/utils/semver.go +++ b/pkg/utils/semver.go @@ -15,9 +15,13 @@ package utils import ( "fmt" + "regexp" + "strconv" "strings" "golang.org/x/mod/semver" + + "github.com/pingcap/errors" ) // NightlyVersionAlias represents latest build of master branch. @@ -74,3 +78,194 @@ func (v Version) IsNightly() bool { func (v Version) String() string { return string(v) } + +type ver struct { + Major, Minor, Patch int + Prerelease []string +} + +func (v *ver) Compare(other *ver) int { + if d := compareSegment(v.Major, other.Major); d != 0 { + return d + } + if d := compareSegment(v.Minor, other.Minor); d != 0 { + return d + } + if d := compareSegment(v.Patch, other.Patch); d != 0 { + return d + } + + return comparePrerelease(v.Prerelease, other.Prerelease) +} + +// Constraint for semver +type Constraint struct { + // [min, max) + min ver // min ver + max ver // max ver +} + +var ( + constraintRegex = regexp.MustCompile( + `^(?P[~^])?v?(?P0|[1-9]\d*)(?:\.(?P0|[1-9]\d*|[x*]))?(?:\.(?P0|[1-9]\d*|[x*]))?(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`, + ) + + versionRegexp = regexp.MustCompile( + `^v?(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`, + ) +) + +// NewConstraint creates a constraint to check whether a semver is valid. +// Only support ^ and ~ and x|X|* +func NewConstraint(raw string) (*Constraint, error) { + result := MatchGroups(constraintRegex, strings.ToLower(strings.TrimSpace(raw))) + if len(result) == 0 { + return nil, errors.New("fail to parse version constraint") + } + c := &Constraint{} + defer func() { + if len(c.max.Prerelease) == 0 { + c.max.Prerelease = []string{"0"} + } + }() + + c.min.Major = MustAtoI(result["major"]) + c.max.Major = c.min.Major + + if minor := result["minor"]; minor == "x" || minor == "*" { + c.max.Major = c.min.Major + 1 + return c, nil + } else if minor != "" { + c.min.Minor = MustAtoI(result["minor"]) + c.max.Minor = c.min.Minor + } + + if patch := result["patch"]; patch == "x" || patch == "*" { + c.max.Minor = c.min.Minor + 1 + return c, nil + } else if patch != "" { + c.min.Patch = MustAtoI(result["patch"]) + c.max.Patch = c.min.Patch + } + + if prerelease := result["prerelease"]; prerelease != "" { + c.min.Prerelease = strings.Split(prerelease, ".") + } else { + c.min.Prerelease = []string{} + } + + c.max.Prerelease = append([]string(nil), c.min.Prerelease...) + + if constraint := result["constraint"]; constraint == "~" { + // ~x.y.z -> >=x.y.z =0.0.z + c.max.Patch++ + } else { + // ^0.y.z -> >=0.y.z <0.(y+1).0 + c.max.Minor++ + c.max.Patch = 0 + } + } else { + // ^x.y.z -> >=x.y.z <(x+1).0.0 + c.max.Major++ + c.max.Minor = 0 + c.max.Patch = 0 + } + } else if l := len(c.max.Prerelease); l > 0 { + c.max.Prerelease[l-1] += " " + } else { + c.max.Patch++ + } + + return c, nil +} + +// Check checks whether a version is satisfies the constraint +func (c *Constraint) Check(v string) bool { + result := MatchGroups(versionRegexp, strings.ToLower(strings.TrimSpace(v))) + if len(result) == 0 { + return false + } + + major := MustAtoI(result["major"]) + minor := MustAtoI(result["minor"]) + patch := MustAtoI(result["patch"]) + version := &ver{ + Major: major, + Minor: minor, + Patch: patch, + Prerelease: []string{}, + } + if pre := result["prerelease"]; pre != "" { + version.Prerelease = strings.Split(pre, ".") + } + return c.min.Compare(version) <= 0 && c.max.Compare(version) > 0 +} + +func compareSegment(v, o int) int { + if v < o { + return -1 + } + if v > o { + return 1 + } + return 0 +} + +// 1, -1, 0 means A>B, A len(preA) { + return -1 + } + return 0 +} + +func compareAlphaNum(a, b string) int { + if a == b { + return 0 + } + iA, errA := strconv.Atoi(a) + iB, errB := strconv.Atoi(b) + if errA != nil && errB != nil { + if a > b { + return 1 + } + return -1 + } + // Numeric identifiers always have lower precedence than non-numeric identifiers. + if errA != nil { + return 1 + } + if errB != nil { + return -1 + } + if iA > iB { + return 1 + } + return -1 +} diff --git a/pkg/utils/semver_test.go b/pkg/utils/semver_test.go index c81661d25d..582ce9a7c9 100644 --- a/pkg/utils/semver_test.go +++ b/pkg/utils/semver_test.go @@ -33,3 +33,53 @@ func (s *TestSemverSuite) TestVersion(c *C) { c.Assert(Version("nightly").IsNightly(), IsTrue) c.Assert(Version("v1.2.3").String(), Equals, "v1.2.3") } + +func (s *TestSemverSuite) TestConstraint(c *C) { + cases := []struct { + constraint string + version string + match bool + }{ + {"^4", "4.1.0", true}, + {"4", "4.0.0", true}, + {"4.0", "4.0.0", true}, + {"~4.0", "4.0.5", true}, + {"4.1.x", "4.1.0", true}, + {"4.1.x", "4.1.5", true}, + {"4.x.0", "4.5.0", true}, + {"4.x.0", "4.5.2", true}, + {"4.x.x", "4.5.2", true}, + {"4.3.2-0", "4.3.2", false}, + {"^1.1.0", "1.1.1", true}, + {"~1.1.0", "1.1.1", true}, + {"~1.1.0", "1.2.0", false}, + {"^1.x.x", "1.1.1", true}, + {"^2.x.x", "1.1.1", false}, + {"^1.x.x", "2.1.1", false}, + {"^1.x.x", "1.1.1-beta1", true}, + {"^1.1.2-alpha", "1.2.1-beta.1", true}, + {"^1.2.x", "1.2.1-beta.1", true}, + {"~1.1.1-beta", "1.1.1-alpha", false}, + {"~1.1.1-beta", "1.1.1-beta.1", true}, + {"~1.1.1-beta", "1.1.1", true}, + {"~1.2.3", "1.2.5", true}, + {"~1.2.3", "1.2.2", false}, + {"~1.2.3", "1.3.2", false}, + {"~1.1.*", "1.2.3", false}, + {"~1.3.0", "2.4.5", false}, + {"^4.0", "5.0.0-rc", false}, + {"^4.0-rc", "5.0.0-rc", false}, + {"4.0.0-rc", "4.0.0-rc", true}, + {"~4.0.0-rc", "4.0.0-rc.1", true}, + {"^4", "v5.0.0-20210408", false}, + {"^4.*.*", "5.0.0-0", false}, + {"5.*.*", "5.0.0-0", false}, + {"^4.0.0-1", "4.0.0-1", true}, + {"4.0.0-1", "4.0.0-1", true}, + } + for _, cas := range cases { + cons, err := NewConstraint(cas.constraint) + c.Assert(err, IsNil) + c.Assert(cons.Check(cas.version), Equals, cas.match) + } +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index df92e62f8c..b72fc72a3f 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -40,3 +40,9 @@ func IsFlagSetByUser(flagSet *pflag.FlagSet, flagName string) bool { }) return setByUser } + +// MustAtoI calls strconv.Atoi and ignores error +func MustAtoI(a string) int { + v, _ := strconv.Atoi(a) + return v +}