Skip to content

Commit

Permalink
*: add version check before start (pingcap#311)
Browse files Browse the repository at this point in the history
* add version check

* Update pkg/utils/version.go

Co-authored-by: kennytm <[email protected]>

* fix test

* add flag to control check

* address comment

* fix ci

* remove DS_Store

Co-authored-by: kennytm <[email protected]>
Co-authored-by: Neil Shen <[email protected]>
  • Loading branch information
3 people committed Jun 5, 2020
1 parent abb9f88 commit 7c5c187
Show file tree
Hide file tree
Showing 12 changed files with 279 additions and 47 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ build:
build_for_integration_test: failpoint-enable
(GO111MODULE=on go test -c -cover -covermode=count \
-coverpkg=$(BR_PKG)/... \
-ldflags '$(LDFLAGS)'\
-o bin/br.test && \
GO111MODULE=on go build ${RACEFLAG} -o bin/locker tests/br_key_locked/*.go && \
GO111MODULE=on go build ${RACEFLAG} -o bin/gc tests/br_z_gc_safepoint/*.go && \
Expand Down
24 changes: 10 additions & 14 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,21 @@ module github.com/pingcap/br
go 1.13

require (
cloud.google.com/go/storage v1.4.0
github.com/aws/aws-sdk-go v1.26.1
github.com/cheggaaa/pb/v3 v3.0.1
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/fatih/color v1.9.0 // indirect
github.com/fsouza/fake-gcs-server v1.15.0
github.com/go-sql-driver/mysql v1.4.1
cloud.google.com/go/storage v1.5.0
github.com/aws/aws-sdk-go v1.30.24
github.com/cheggaaa/pb/v3 v3.0.4
github.com/coreos/go-semver v0.3.0
github.com/fsouza/fake-gcs-server v1.17.0
github.com/go-sql-driver/mysql v1.5.0
github.com/gogo/protobuf v1.3.1
github.com/google/btree v1.0.0
github.com/google/uuid v1.1.1
github.com/klauspost/cpuid v1.2.0 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/montanaflynn/stats v0.5.0 // indirect
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
github.com/pingcap/failpoint v0.0.0-20200506114213-c17f16071c53
github.com/pingcap/kvproto v0.0.0-20200506072111-3a6b8b9cb29e
github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29
github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
github.com/pingcap/parser v3.1.0-beta.2.0.20200425032215-994651e9b6df+incompatible
github.com/pingcap/pd/v3 v3.1.1-0.20200426091027-e639f0b1e62b
Expand All @@ -35,10 +32,9 @@ require (
github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
go.opencensus.io v0.22.2 // indirect
go.uber.org/multierr v1.5.0
go.uber.org/zap v1.14.1
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
google.golang.org/api v0.14.0
google.golang.org/grpc v1.25.1
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6
google.golang.org/api v0.15.0
google.golang.org/grpc v1.26.0
)
84 changes: 64 additions & 20 deletions go.sum

Large diffs are not rendered by default.

8 changes: 7 additions & 1 deletion pkg/conn/conn.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ func NewMgr(
tlsConf *tls.Config,
securityOption pd.SecurityOption,
storeBehavior StoreBehavior,
checkRequirements bool,
) (*Mgr, error) {
addrs := strings.Split(pdAddrs, ",")

Expand All @@ -178,7 +179,6 @@ func NewMgr(
}
processedAddrs = append(processedAddrs, addr)
_, failure = pdRequest(ctx, addr, clusterVersionPrefix, cli, http.MethodGet, nil)
// TODO need check cluster version >= 3.1 when br release
if failure == nil {
break
}
Expand All @@ -197,6 +197,12 @@ func NewMgr(
log.Error("fail to create pd client", zap.Error(err))
return nil, err
}
if checkRequirements {
err = utils.CheckClusterVersion(ctx, pdClient)
if err != nil {
return nil, err
}
}
log.Info("new mgr", zap.String("pdAddrs", pdAddrs))

// Check live tikv.
Expand Down
2 changes: 1 addition & 1 deletion pkg/task/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
if err != nil {
return err
}
mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash)
mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash, cfg.CheckRequirements)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/task/backup_raw.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func RunBackupRaw(c context.Context, g glue.Glue, cmdName string, cfg *RawKvConf
if err != nil {
return err
}
mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash)
mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash, cfg.CheckRequirements)
if err != nil {
return err
}
Expand Down
25 changes: 18 additions & 7 deletions pkg/task/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,11 @@ const (
flagDatabase = "db"
flagTable = "table"

flagRateLimit = "ratelimit"
flagRateLimitUnit = "ratelimit-unit"
flagConcurrency = "concurrency"
flagChecksum = "checksum"
flagRateLimit = "ratelimit"
flagRateLimitUnit = "ratelimit-unit"
flagConcurrency = "concurrency"
flagChecksum = "checksum"
flagCheckRequirement = "check-requirements"
)

// TLSConfig is the common configuration for TLS connection.
Expand Down Expand Up @@ -88,8 +89,9 @@ type Config struct {
// LogProgress is true means the progress bar is printed to the log instead of stdout.
LogProgress bool `json:"log-progress" toml:"log-progress"`

CaseSensitive bool `json:"case-sensitive" toml:"case-sensitive"`
Filter filter.Rules `json:"black-white-list" toml:"black-white-list"`
CaseSensitive bool `json:"case-sensitive" toml:"case-sensitive"`
CheckRequirements bool `json:"check-requirements" toml:"check-requirements"`
Filter filter.Rules `json:"black-white-list" toml:"black-white-list"`
}

// DefineCommonFlags defines the flags common to all BRIE commands.
Expand All @@ -113,6 +115,9 @@ func DefineCommonFlags(flags *pflag.FlagSet) {
flags.Uint64(flagRateLimitUnit, utils.MB, "The unit of rate limit")
_ = flags.MarkHidden(flagRateLimitUnit)

flags.Bool(flagCheckRequirement, true,
"Whether start version check before execute command")

storage.DefineFlags(flags)
}

Expand Down Expand Up @@ -200,6 +205,11 @@ func (cfg *Config) ParseFromFlags(flags *pflag.FlagSet) error {
cfg.Filter.DoDBs = []string{db}
}
}
checkRequirements, err := flags.GetBool(flagCheckRequirement)
if err != nil {
return errors.Trace(err)
}
cfg.CheckRequirements = checkRequirements

if err := cfg.BackendOptions.ParseFromFlags(flags); err != nil {
return err
Expand All @@ -214,6 +224,7 @@ func newMgr(
pds []string,
tlsConfig TLSConfig,
storeBehavior conn.StoreBehavior,
checkRequirements bool,
) (*conn.Mgr, error) {
var (
tlsConf *tls.Config
Expand All @@ -240,7 +251,7 @@ func newMgr(
if err != nil {
return nil, err
}
return conn.NewMgr(ctx, g, pdAddress, store.(tikv.Storage), tlsConf, securityOption, storeBehavior)
return conn.NewMgr(ctx, g, pdAddress, store.(tikv.Storage), tlsConf, securityOption, storeBehavior, checkRequirements)
}

// GetStorage gets the storage backend from the config.
Expand Down
4 changes: 2 additions & 2 deletions pkg/task/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
ctx, cancel := context.WithCancel(c)
defer cancel()

mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash)
mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash, cfg.CheckRequirements)
if err != nil {
return err
}
Expand Down Expand Up @@ -519,7 +519,7 @@ func RunRestoreTiflashReplica(c context.Context, g glue.Glue, cmdName string, cf
ctx, cancel := context.WithCancel(c)
defer cancel()

mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash)
mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash, cfg.CheckRequirements)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/task/restore_raw.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
ctx, cancel := context.WithCancel(c)
defer cancel()

mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.ErrorOnTiFlash)
mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.ErrorOnTiFlash, cfg.CheckRequirements)
if err != nil {
return err
}
Expand Down
64 changes: 64 additions & 0 deletions pkg/utils/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,15 @@ package utils

import (
"bytes"
"context"
"fmt"
"runtime"
"strings"

"github.com/coreos/go-semver/semver"
"github.com/pingcap/errors"
"github.com/pingcap/log"
pd "github.com/pingcap/pd/v3/client"
"github.com/pingcap/tidb/util/israce"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
Expand Down Expand Up @@ -54,3 +59,62 @@ func LogArguments(cmd *cobra.Command) {
})
log.Info("arguments", fields...)
}

var minTiKVVersion *semver.Version = semver.New("3.1.0-beta.2")
var incompatibleTiKVMajor3 *semver.Version = semver.New("3.1.0")
var incompatibleTiKVMajor4 *semver.Version = semver.New("4.0.0-rc.1")

func removeV(v string) string {
return strings.TrimPrefix(v, "v")
}

// CheckClusterVersion check TiKV version.
func CheckClusterVersion(ctx context.Context, client pd.Client) error {
BRVersion, err := semver.NewVersion(removeV(BRReleaseVersion))
if err != nil {
return err
}
stores, err := client.GetAllStores(ctx, pd.WithExcludeTombstone())
if err != nil {
return err
}
for _, s := range stores {
tikvVersion, err := semver.NewVersion(removeV(s.Version))
if err != nil {
return err
}

if tikvVersion.Compare(*minTiKVVersion) < 0 {
return errors.Errorf("TiKV node %s version %s don't support BR, please upgrade cluster to %s",
s.Address, removeV(s.Version), BRReleaseVersion)
}

if tikvVersion.Major != BRVersion.Major {
return errors.Errorf("TiKV node %s version %s and BR %s major version mismatch, please use the same version of BR",
s.Address, removeV(s.Version), BRReleaseVersion)
}

// BR(https://github.com/pingcap/br/pull/233) and TiKV(https://github.com/tikv/tikv/pull/7241) have breaking changes
// if BR include #233 and TiKV not include #7241, BR will panic TiKV during restore
// These incompatible version is 3.1.0 and 4.0.0-rc.1
if tikvVersion.Major == 3 {
if tikvVersion.Compare(*incompatibleTiKVMajor3) < 0 && BRVersion.Compare(*incompatibleTiKVMajor3) >= 0 {
return errors.Errorf("TiKV node %s version %s and BR %s version mismatch, please use the same version of BR",
s.Address, removeV(s.Version), BRReleaseVersion)
}
}

if tikvVersion.Major == 4 {
if tikvVersion.Compare(*incompatibleTiKVMajor4) < 0 && BRVersion.Compare(*incompatibleTiKVMajor4) >= 0 {
return errors.Errorf("TiKV node %s version %s and BR %s version mismatch, please use the same version of BR",
s.Address, removeV(s.Version), BRReleaseVersion)
}
}

if tikvVersion.Compare(*BRVersion) > 0 {
log.Warn(fmt.Sprintf("BR version is too old, please consider use version %s of BR", removeV(s.Version)))
break
}
}
return nil
}
108 changes: 108 additions & 0 deletions pkg/utils/version_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package utils

import (
"context"

"github.com/coreos/go-semver/semver"
"github.com/pingcap/check"
"github.com/pingcap/kvproto/pkg/metapb"
pd "github.com/pingcap/pd/v3/client"
)

type versionSuite struct{}

var _ = check.Suite(&versionSuite{})

type mockPDClient struct {
pd.Client
getAllStores func() []*metapb.Store
}

func (m *mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) {
if m.getAllStores != nil {
return m.getAllStores(), nil
}
return []*metapb.Store{}, nil
}

func (s *versionSuite) TestCheckClusterVersion(c *check.C) {
mock := mockPDClient{
Client: nil,
}

{
BRReleaseVersion = "v3.1.0-beta.2"
mock.getAllStores = func() []*metapb.Store {
return []*metapb.Store{{Version: minTiKVVersion.String()}}
}
err := CheckClusterVersion(context.Background(), &mock)
c.Assert(err, check.IsNil)
}

{
BRReleaseVersion = "v3.1.0-beta.2"
mock.getAllStores = func() []*metapb.Store {
// TiKV is too lower to support BR
return []*metapb.Store{{Version: `v2.1.0`}}
}
err := CheckClusterVersion(context.Background(), &mock)
c.Assert(err, check.ErrorMatches, "TiKV .* don't support BR, please upgrade cluster .*")
}

{
BRReleaseVersion = "v3.1.0"
mock.getAllStores = func() []*metapb.Store {
// TiKV v3.1.0-beta.2 is incompatible with BR v3.1.0
return []*metapb.Store{{Version: minTiKVVersion.String()}}
}
err := CheckClusterVersion(context.Background(), &mock)
c.Assert(err, check.ErrorMatches, "TiKV .* mismatch, please .*")
}

{
BRReleaseVersion = "v3.1.0"
mock.getAllStores = func() []*metapb.Store {
// TiKV v4.0.0-rc major version mismatch with BR v3.1.0
return []*metapb.Store{{Version: "v4.0.0-rc"}}
}
err := CheckClusterVersion(context.Background(), &mock)
c.Assert(err, check.ErrorMatches, "TiKV .* major version mismatch, please .*")
}

{
BRReleaseVersion = "v4.0.0-rc.2"
mock.getAllStores = func() []*metapb.Store {
// TiKV v4.0.0-rc.2 is incompatible with BR v4.0.0-beta.1
return []*metapb.Store{{Version: "v4.0.0-beta.1"}}
}
err := CheckClusterVersion(context.Background(), &mock)
c.Assert(err, check.ErrorMatches, "TiKV .* mismatch, please .*")
}

{
BRReleaseVersion = "v4.0.0-rc.2"
mock.getAllStores = func() []*metapb.Store {
// TiKV v4.0.0-rc.1 with BR v4.0.0-rc.2 is ok
return []*metapb.Store{{Version: "v4.0.0-rc.1"}}
}
err := CheckClusterVersion(context.Background(), &mock)
c.Assert(err, check.IsNil)
}

{
BRReleaseVersion = "v4.0.0-rc.1"
mock.getAllStores = func() []*metapb.Store {
// TiKV v4.0.0-rc.2 with BR v4.0.0-rc.1 is ok
return []*metapb.Store{{Version: "v4.0.0-rc.2"}}
}
err := CheckClusterVersion(context.Background(), &mock)
c.Assert(err, check.IsNil)
}
}

func (s *versionSuite) TestCompareVersion(c *check.C) {
c.Assert(semver.New("4.0.0-rc").Compare(*semver.New("4.0.0-rc.2")), check.Equals, -1)
c.Assert(semver.New("4.0.0-beta.3").Compare(*semver.New("4.0.0-rc.2")), check.Equals, -1)
c.Assert(semver.New("4.0.0-rc.1").Compare(*semver.New("4.0.0")), check.Equals, -1)
c.Assert(semver.New("4.0.0-beta.1").Compare(*semver.New("4.0.0")), check.Equals, -1)
}
2 changes: 2 additions & 0 deletions tools/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,10 @@ github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/ryancurrah/gomodguard v1.0.2/go.mod h1:9T/Cfuxs5StfsocWr4WzDL36HqnX0fVb9d5fSEaLhoE=
github.com/ryancurrah/gomodguard v1.0.4 h1:oCreMAt9GuFXDe9jW4HBpc3GjdX3R/sUEcLAGh1zPx8=
github.com/ryancurrah/gomodguard v1.0.4/go.mod h1:9T/Cfuxs5StfsocWr4WzDL36HqnX0fVb9d5fSEaLhoE=
github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE=
github.com/securego/gosec v0.0.0-20200316084457-7da9f46445fd h1:qB+l4fYZsH78xORC1aqVS0zNmgkQp4rkj2rvfxQMtzc=
github.com/securego/gosec v0.0.0-20200316084457-7da9f46445fd/go.mod h1:NurAFZsWJAEZjogSwdVPlHkOZB3DOAU7gsPP8VFZCHc=
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc=
Expand Down

0 comments on commit 7c5c187

Please sign in to comment.