From 4d5347df369991ee99ad203085d18c0c2eacc9ae Mon Sep 17 00:00:00 2001 From: JaySon Date: Tue, 23 Mar 2021 01:33:36 -0500 Subject: [PATCH] Omitting deprecated `"users.*"` settings for TiFlash (#1211) --- .../instance/tiflash_proxy_config.go | 2 +- components/playground/playground.go | 2 +- pkg/cluster/spec/spec_test.go | 53 ++++++++++++++++ pkg/cluster/spec/tiflash.go | 60 +++++++++++-------- 4 files changed, 91 insertions(+), 26 deletions(-) diff --git a/components/playground/instance/tiflash_proxy_config.go b/components/playground/instance/tiflash_proxy_config.go index 133c095e5d..2d9f25629a 100644 --- a/components/playground/instance/tiflash_proxy_config.go +++ b/components/playground/instance/tiflash_proxy_config.go @@ -51,7 +51,7 @@ func writeTiFlashProxyConfig(w io.Writer, version utils.Version, ip, deployDir s dataDir := fmt.Sprintf("%s/flash", deployDir) logDir := fmt.Sprintf("%s/log", deployDir) var statusAddr string - if semver.Compare(version.String(), "v4.0.5") >= 0 || version.String() == "nightly" { + if semver.Compare(version.String(), "v4.0.5") >= 0 || version.IsNightly() { statusAddr = fmt.Sprintf(`status-addr = "0.0.0.0:%[2]d" advertise-status-addr = "%[1]s:%[2]d"`, ip, proxyStatusPort) } else { diff --git a/components/playground/playground.go b/components/playground/playground.go index 0ff25c58ab..d408953a43 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -685,7 +685,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme `, options.version)) } - if options.version != "nightly" { + if !utils.Version(options.version).IsNightly() { if semver.Compare(options.version, "v3.1.0") < 0 && options.tiflash.Num != 0 { fmt.Println(color.YellowString("Warning: current version %s doesn't support TiFlash", options.version)) options.tiflash.Num = 0 diff --git a/pkg/cluster/spec/spec_test.go b/pkg/cluster/spec/spec_test.go index 05cbc743c6..86e838906d 100644 --- a/pkg/cluster/spec/spec_test.go +++ b/pkg/cluster/spec/spec_test.go @@ -684,6 +684,59 @@ tiflash_servers: } } +func (s *metaSuiteTopo) TestTiFlashUsersSettings(c *C) { + spec := &Specification{} + err := yaml.Unmarshal([]byte(` +tiflash_servers: + - host: 172.16.5.138 + data_dir: /ssd0/tiflash +`), spec) + c.Assert(err, IsNil) + + flashComp := FindComponent(spec, ComponentTiFlash) + instances := flashComp.Instances() + c.Assert(len(instances), Equals, 1) + + // parse using clusterVersion<"v4.0.12" || == "5.0.0-rc" + checkBackwardCompatibility := func(ver string) { + ins := instances[0].(*TiFlashInstance) + dataDir := "/ssd0/tiflash" + cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") + conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash) + c.Assert(err, IsNil) + + // We need an empty string for 'users.default.password' for backward compatibility. Or the TiFlash process will fail to start with older versions + if usersSection, ok := conf["users"]; !ok { + c.Error("Can not get users section") + } else { + if defaultUser, ok := usersSection.(map[string]interface{})["default"]; !ok { + c.Error("Can not get default users section") + } else { + var password = defaultUser.(map[string]interface{})["password"] + c.Assert(password.(string), Equals, "") + } + } + } + checkBackwardCompatibility("v4.0.11") + checkBackwardCompatibility("v5.0.0-rc") + + // parse using clusterVersion>="v4.0.12" + checkWithVersion := func(ver string) { + ins := instances[0].(*TiFlashInstance) + dataDir := "/ssd0/tiflash" + cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") + conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash) + c.Assert(err, IsNil) + + // Those deprecated settings are ignored in newer versions + _, ok := conf["users"] + c.Assert(ok, IsFalse) + } + checkWithVersion("v4.0.12") + checkWithVersion("v5.0.0") + checkWithVersion("nightly") +} + func (s *metaSuiteTopo) TestYAMLAnchor(c *C) { topo := Specification{} err := yaml.UnmarshalStrict([]byte(` diff --git a/pkg/cluster/spec/tiflash.go b/pkg/cluster/spec/tiflash.go index dd68a4c579..093d7fddef 100644 --- a/pkg/cluster/spec/tiflash.go +++ b/pkg/cluster/spec/tiflash.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" + "github.com/pingcap/tiup/pkg/utils" "golang.org/x/mod/semver" "gopkg.in/yaml.v2" ) @@ -337,14 +338,14 @@ func (i *TiFlashInstance) CheckIncorrectConfigs() error { if _, err := checkTiFlashStorageConfig(i.InstanceSpec.(*TiFlashSpec).Config); err != nil { return err } - // no matter storgae.latest.dir is defined or not, return err + // no matter storage.latest.dir is defined or not, return err _, err := isValidStringArray(TiFlashStorageKeyLatestDirs, i.InstanceSpec.(*TiFlashSpec).Config, true) return err } // need to check the configuration after clusterVersion >= v4.0.9. func checkTiFlashStorageConfigWithVersion(clusterVersion string, config map[string]interface{}) (bool, error) { - if semver.Compare(clusterVersion, "v4.0.9") >= 0 || clusterVersion == "nightly" { + if semver.Compare(clusterVersion, "v4.0.9") >= 0 || utils.Version(clusterVersion).IsNightly() { return checkTiFlashStorageConfig(config) } return false, nil @@ -353,9 +354,10 @@ func checkTiFlashStorageConfigWithVersion(clusterVersion string, config map[stri // InitTiFlashConfig initializes TiFlash config file with the configurations in server_configs func (i *TiFlashInstance) initTiFlashConfig(cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}) (map[string]interface{}, error) { var ( - pathConfig string - isStorageDirsDefined bool - err error + pathConfig string + isStorageDirsDefined bool + deprecatedUsersConfig string + err error ) if isStorageDirsDefined, err = checkTiFlashStorageConfigWithVersion(clusterVersion, src); err != nil { return nil, err @@ -367,6 +369,32 @@ func (i *TiFlashInstance) initTiFlashConfig(cfg *scripts.TiFlashScript, clusterV pathConfig = fmt.Sprintf(`path: "%s"`, cfg.DataDir) } + if (semver.Compare(clusterVersion, "v4.0.12") >= 0 && semver.Compare(clusterVersion, "v5.0.0-rc") != 0) || utils.Version(clusterVersion).IsNightly() { + // For v4.0.12 or later, 5.0.0 or later, TiFlash can ignore these `user.*`, `quotas.*` settings + deprecatedUsersConfig = "#" + } else { + // These settings is required when the version is earlier than v4.0.12 and v5.0.0 + deprecatedUsersConfig = ` + quotas.default.interval.duration: 3600 + quotas.default.interval.errors: 0 + quotas.default.interval.execution_time: 0 + quotas.default.interval.queries: 0 + quotas.default.interval.read_rows: 0 + quotas.default.interval.result_rows: 0 + users.default.password: "" + users.default.profile: "default" + users.default.quota: "default" + users.default.networks.ip: "::/0" + users.readonly.password: "" + users.readonly.profile: "readonly" + users.readonly.quota: "default" + users.readonly.networks.ip: "::/0" + profiles.default.load_balancing: "random" + profiles.default.use_uncompressed_cache: 0 + profiles.readonly.readonly: 1 +` + } + topo := Specification{} err = yaml.Unmarshal([]byte(fmt.Sprintf(` @@ -396,26 +424,10 @@ server_configs: logger.size: "1000M" application.runAsDaemon: true raft.pd_addr: "%[9]s" - quotas.default.interval.duration: 3600 - quotas.default.interval.errors: 0 - quotas.default.interval.execution_time: 0 - quotas.default.interval.queries: 0 - quotas.default.interval.read_rows: 0 - quotas.default.interval.result_rows: 0 - users.default.password: "" - users.default.profile: "default" - users.default.quota: "default" - users.default.networks.ip: "::/0" - users.readonly.password: "" - users.readonly.profile: "readonly" - users.readonly.quota: "default" - users.readonly.networks.ip: "::/0" - profiles.default.load_balancing: "random" profiles.default.max_memory_usage: 0 - profiles.default.use_uncompressed_cache: 0 - profiles.readonly.readonly: 1 + %[12]s `, pathConfig, cfg.LogDir, cfg.TCPPort, cfg.HTTPPort, cfg.TiDBStatusAddrs, cfg.IP, cfg.FlashServicePort, - cfg.StatusPort, cfg.PDAddrs, cfg.DeployDir, cfg.TmpDir)), &topo) + cfg.StatusPort, cfg.PDAddrs, cfg.DeployDir, cfg.TmpDir, deprecatedUsersConfig)), &topo) if err != nil { return nil, err @@ -456,7 +468,7 @@ func (i *TiFlashInstance) InitTiFlashLearnerConfig(cfg *scripts.TiFlashScript, c firstDataDir := strings.Split(cfg.DataDir, ",")[0] - if semver.Compare(clusterVersion, "v4.0.5") >= 0 || clusterVersion == "nightly" { + if semver.Compare(clusterVersion, "v4.0.5") >= 0 || utils.Version(clusterVersion).IsNightly() { statusAddr = fmt.Sprintf(`server.status-addr: "0.0.0.0:%[2]d" server.advertise-status-addr: "%[1]s:%[2]d"`, cfg.IP, cfg.FlashProxyStatusPort) } else {