From 890bb2087f49f2dece232b503a36939a41de3dbd Mon Sep 17 00:00:00 2001 From: srstack Date: Fri, 26 Nov 2021 11:46:05 +0800 Subject: [PATCH 01/28] add tls comand --- components/cluster/command/root.go | 1 + components/cluster/command/tls.go | 64 +++++++++++++ pkg/cluster/manager/builder.go | 147 ++++++++++++++++++++++++++++- pkg/cluster/manager/reload.go | 1 + pkg/cluster/manager/tls.go | 97 +++++++++++++++++++ 5 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 components/cluster/command/tls.go create mode 100644 pkg/cluster/manager/tls.go diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go index e1dfa74524..a5ec91869e 100644 --- a/components/cluster/command/root.go +++ b/components/cluster/command/root.go @@ -207,6 +207,7 @@ func init() { newTelemetryCmd(), newReplayCmd(), newTemplateCmd(), + newTLSCmd(), ) } diff --git a/components/cluster/command/tls.go b/components/cluster/command/tls.go new file mode 100644 index 0000000000..11eae84f37 --- /dev/null +++ b/components/cluster/command/tls.go @@ -0,0 +1,64 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "strings" + + perrs "github.com/pingcap/errors" + "github.com/spf13/cobra" +) + +func newTLSCmd() *cobra.Command { + var ( + skipRestart bool // skip restart cluster, but the tls option does not work + reloadCertificate bool // reload certificate when the cluster enable encrypted communication + cleanCertificate bool // cleanup certificate when the cluster disable encrypted communication + enableTLS bool + ) + + cmd := &cobra.Command{ + Use: "tls ", + Short: "Enable/Disable TLS between TiDB components", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 2 { + return cmd.Help() + } + + if err := validRoles(gOpt.Roles); err != nil { + return err + } + clusterName := args[0] + clusterReport.ID = scrubClusterName(clusterName) + teleCommand = append(teleCommand, scrubClusterName(clusterName)) + + switch strings.ToLower(args[1]) { + case "enable": + enableTLS = true + case "disable": + enableTLS = false + default: + return perrs.New("enable or disable must be specified at least one") + } + + return cm.TLS(clusterName, gOpt, enableTLS, skipRestart, cleanCertificate, reloadCertificate) + }, + } + + cmd.Flags().BoolVar(&cleanCertificate, "clean-certificate", false, "Clean up the certificate file if it already exists when disable encrypted communication") + cmd.Flags().BoolVar(&reloadCertificate, "reload-certificate", false, "Reload the certificate file if it already exists when enable encrypted communication") + cmd.Flags().BoolVar(&skipRestart, "skip-restart", false, "Only refresh configuration and load certificate to remote, but do not restart services") + + return cmd +} diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 8dc269eb2e..f0e8b1e43d 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -319,7 +319,7 @@ func buildScaleOutTask( base.Version, gOpt, p, - ) + ) if err != nil { return nil, err } @@ -699,3 +699,148 @@ func buildDownloadSparkTask(inst spec.Instance, gOpt operator.Options) *task.Ste BuildAsStep(fmt.Sprintf(" - Download %s: (%s/%s)", spec.ComponentSpark, inst.OS(), inst.Arch())) } + +// buildTLSTask create enable/disable tls task +func buildTLSTask( + m *Manager, + name string, + metadata spec.Metadata, + gOpt operator.Options, + p *tui.SSHConnectionProps, + skipRestart bool, +) (task.Task, error) { + var ( + pushCertificateTasks []task.Task // tasks which are used to copy certificate to remote host + refreshConfigTasks []task.Task // tasks which are used to refresh configuration + ) + + topo := metadata.GetTopology() + base := metadata.GetBaseMeta() + specManager := m.specManager + + sshType := topo.BaseTopo().GlobalOptions.SSHType + + var iterErr error + + if topo.BaseTopo().GlobalOptions.TLSEnabled { + // copy certificate to remote host + topo.IterInstance(func(inst spec.Instance) { + + deployDir := spec.Abs(base.User, inst.DeployDir()) + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + + // generate and transfer tls cert for instance + tb := task.NewBuilder(gOpt.DisplayMode). + UserSSH( + inst.GetHost(), + inst.GetSSHPort(), + base.User, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + p.Password, + p.IdentityFile, + p.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + sshType, + ).Mkdir(base.User, inst.GetHost(), deployDir, tlsDir) + + ca, err := crypto.ReadCA( + name, + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), + ) + if err != nil { + iterErr = err + return + } + t := tb.TLSCert( + inst.GetHost(), + inst.ComponentName(), + inst.Role(), + inst.GetMainPort(), + ca, + meta.DirPaths{ + Deploy: deployDir, + Cache: m.specManager.Path(name, spec.TempConfigPath), + }).Build() + + pushCertificateTasks = append(pushCertificateTasks, t) + }) + } + + if iterErr != nil { + return nil, iterErr + } + + hasImported := false + topo.IterInstance(func(inst spec.Instance) { + deployDir := spec.Abs(base.User, inst.DeployDir()) + // data dir would be empty for components which don't need it + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(base.User, inst.LogDir()) + + // Download and copy the latest component to remote if the cluster is imported from Ansible + tb := task.NewBuilder(gOpt.DisplayMode) + if inst.IsImported() { + switch compName := inst.ComponentName(); compName { + case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: + version := m.bindVersion(compName, base.Version) + tb.Download(compName, inst.OS(), inst.Arch(), version). + CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) + } + hasImported = true + } + + // Refresh all configuration + t := tb.InitConfig(name, + base.Version, + m.specManager, + inst, + base.User, + true, // always ignore config check result in scale out + meta.DirPaths{ + Deploy: deployDir, + Data: dataDirs, + Log: logDir, + Cache: specManager.Path(name, spec.TempConfigPath), + }, + ).Build() + refreshConfigTasks = append(refreshConfigTasks, t) + }) + + // handle dir scheme changes + if hasImported { + if err := spec.HandleImportPathMigration(name); err != nil { + return task.NewBuilder(gOpt.DisplayMode).Build(), err + } + } + + builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) + if err != nil { + return nil, err + } + + builder. + Parallel(false, pushCertificateTasks...). + Parallel(false, refreshConfigTasks...). + Func("Save meta", func(_ context.Context) error { + return m.specManager.SaveMeta(name, metadata) + }) + + if !skipRestart { + tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) + if err != nil { + return nil, err + } + builder.Func("Restart Cluster", func(ctx context.Context) error { + return operator.Restart(ctx, topo, gOpt, tlsCfg) + }) + } + + return builder.Build(), nil +} diff --git a/pkg/cluster/manager/reload.go b/pkg/cluster/manager/reload.go index 6d9bdafa8e..304252ef9b 100644 --- a/pkg/cluster/manager/reload.go +++ b/pkg/cluster/manager/reload.go @@ -74,6 +74,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo topo := metadata.GetTopology() base := metadata.GetBaseMeta() + // monitor uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch noAgentHosts := set.NewStringSet() topo.IterInstance(func(inst spec.Instance) { diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go new file mode 100644 index 0000000000..368eca853f --- /dev/null +++ b/pkg/cluster/manager/tls.go @@ -0,0 +1,97 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + "fmt" + + "github.com/joomcode/errorx" + perrs "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/cluster/clusterutil" + "github.com/pingcap/tiup/pkg/cluster/ctxt" + "github.com/pingcap/tiup/pkg/cluster/executor" + operator "github.com/pingcap/tiup/pkg/cluster/operation" + "github.com/pingcap/tiup/pkg/cluster/spec" + "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/tui" + "golang.org/x/mod/semver" +) + +// TLS set cluster enable/disable encrypt communication by tls +func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, cleanCertificate, reloadCertificate bool) error { + if err := clusterutil.ValidateClusterNameOrError(name); err != nil { + return err + } + + // check locked + if err := m.specManager.ScaleOutLockedErr(name); err != nil { + return err + } + + metadata, err := m.meta(name) + if err != nil { + return err + } + + topo := metadata.GetTopology() + base := metadata.GetBaseMeta() + + // set tls_enabled + topo.BaseTopo().GlobalOptions.TLSEnabled = enable + + // check tiflash version + if clusterSpec, ok := topo.(*spec.Specification); ok { + if clusterSpec.GlobalOptions.TLSEnabled && + semver.Compare(base.Version, "v4.0.5") < 0 && + len(clusterSpec.TiFlashServers) > 0 { + return fmt.Errorf("TiFlash %s is not supported in TLS enabled cluster", base.Version) + } + } + + var ( + sshProxyProps *tui.SSHConnectionProps = &tui.SSHConnectionProps{} + ) + if gOpt.SSHType != executor.SSHTypeNone { + var err error + if len(gOpt.SSHProxyHost) != 0 { + if sshProxyProps, err = tui.ReadIdentityFileOrPassword(gOpt.SSHProxyIdentity, gOpt.SSHProxyUsePassword); err != nil { + return err + } + } + } + + // Build the tls tasks + t, err := buildTLSTask( + m, name, metadata, gOpt, sshProxyProps, skipRestart) + if err != nil { + return err + } + + if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + if errorx.Cast(err) != nil { + // FIXME: Map possible task errors and give suggestions. + return err + } + return perrs.Trace(err) + } + + if enable { + log.Infof("Enable cluster `%s` TLS between TiDB components successfully", name) + } else { + log.Infof("Disable cluster `%s` TLS between TiDB components successfully", name) + } + + return nil +} From 042a1e70f411f06805d5631ee8ccfefce3ec38b6 Mon Sep 17 00:00:00 2001 From: srstack Date: Fri, 26 Nov 2021 12:44:28 +0800 Subject: [PATCH 02/28] add tls func --- pkg/cluster/manager/builder.go | 2 ++ pkg/cluster/manager/cacert.go | 43 ++++++++++++++++++++++++++++++++++ pkg/cluster/manager/deploy.go | 19 +-------------- pkg/cluster/manager/tls.go | 35 ++++++++++++++++++++++++++- 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index f0e8b1e43d..3c26b601e4 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -844,3 +844,5 @@ func buildTLSTask( return builder.Build(), nil } + + diff --git a/pkg/cluster/manager/cacert.go b/pkg/cluster/manager/cacert.go index 6d623b411e..e2d2e466ea 100644 --- a/pkg/cluster/manager/cacert.go +++ b/pkg/cluster/manager/cacert.go @@ -101,3 +101,46 @@ func genAndSaveClientCert(ca *crypto.CertificateAuthority, name, tlsPath string) return nil } + +// genAndSaveCertificate generate CA and client cert for TLS enabled cluster +func (m *Manager) genAndSaveCertificate(clusterName string, globalOptions *spec.GlobalOptions) (*crypto.CertificateAuthority, error) { + var ca *crypto.CertificateAuthority + if globalOptions.TLSEnabled { + // generate CA + tlsPath := m.specManager.Path(clusterName, spec.TLSCertKeyDir) + if err := utils.CreateDir(tlsPath); err != nil { + return nil, err + } + ca, err := genAndSaveClusterCA(clusterName, tlsPath) + if err != nil { + return nil, err + } + + // generate client cert + if err = genAndSaveClientCert(ca, clusterName, tlsPath); err != nil { + return nil, err + } + } + + return ca, nil +} + +// checkCertificate check if the certificate file exists +// no need to determine whether to enable tls +func (m *Manager) checkCertificate(clusterName string) error { + + tlsFiles := []string{ + m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSCACert), + m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSClientKey), + m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSClientCert), + } + + // check if the file exists + for _, file := range tlsFiles { + if !utils.IsExist(file) { + return perrs.Errorf("TLS file: %s does not exist", file) + } + } + + return nil +} diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index 05eae3e7fa..37f93659ed 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -30,7 +30,6 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" @@ -191,23 +190,7 @@ func (m *Manager) Deploy( globalOptions := base.GlobalOptions // generate CA and client cert for TLS enabled cluster - var ca *crypto.CertificateAuthority - if globalOptions.TLSEnabled { - // generate CA - tlsPath := m.specManager.Path(name, spec.TLSCertKeyDir) - if err := utils.CreateDir(tlsPath); err != nil { - return err - } - ca, err = genAndSaveClusterCA(name, tlsPath) - if err != nil { - return err - } - - // generate client cert - if err = genAndSaveClientCert(ca, name, tlsPath); err != nil { - return err - } - } + ca, err := m.genAndSaveCertificate(name, globalOptions) var iterErr error // error when itering over instances iterErr = nil diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 368eca853f..dfe78b4cfc 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -16,6 +16,7 @@ package manager import ( "context" "fmt" + "os" "github.com/joomcode/errorx" perrs "github.com/pingcap/errors" @@ -49,7 +50,15 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c base := metadata.GetBaseMeta() // set tls_enabled - topo.BaseTopo().GlobalOptions.TLSEnabled = enable + globalOptions := topo.BaseTopo().GlobalOptions + globalOptions.TLSEnabled = enable + + // load certificate file + if enable { + if err := loadCertificate(m, name, globalOptions, reloadCertificate); err != nil { + return err + } + } // check tiflash version if clusterSpec, ok := topo.(*spec.Specification); ok { @@ -87,6 +96,13 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c return perrs.Trace(err) } + if !enable { + // the cleanCertificate parameter will only take effect when enable is false + if cleanCertificate { + os.RemoveAll(m.specManager.Path(name, spec.TLSCertKeyDir)) + } + } + if enable { log.Infof("Enable cluster `%s` TLS between TiDB components successfully", name) } else { @@ -95,3 +111,20 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c return nil } + +// loadCertificate +// certificate file exists and reload is true +// will reload certificate file +func loadCertificate(m *Manager, clusterName string, globalOptions *spec.GlobalOptions, reload bool) error { + + err := m.checkCertificate(clusterName) + + // no need to reload and the file already exists + if !reload && err == nil { + return nil + } + + _, err = m.genAndSaveCertificate(clusterName, globalOptions) + + return err +} From ca4303cd44c9e12e78859e82cb74b9fb9ff90a79 Mon Sep 17 00:00:00 2001 From: srstack Date: Fri, 26 Nov 2021 17:15:23 +0800 Subject: [PATCH 03/28] add tls command --- pkg/cluster/manager/builder.go | 11 ++++------- pkg/cluster/manager/cacert.go | 2 -- pkg/cluster/manager/deploy.go | 3 +++ pkg/cluster/manager/tls.go | 19 ++++++++++++++++++- pkg/cluster/spec/spec.go | 10 +++++++++- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 3c26b601e4..d59276a70b 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -319,7 +319,7 @@ func buildScaleOutTask( base.Version, gOpt, p, - ) + ) if err != nil { return nil, err } @@ -725,7 +725,6 @@ func buildTLSTask( if topo.BaseTopo().GlobalOptions.TLSEnabled { // copy certificate to remote host topo.IterInstance(func(inst spec.Instance) { - deployDir := spec.Abs(base.User, inst.DeployDir()) tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) @@ -826,11 +825,11 @@ func buildTLSTask( } builder. - Parallel(false, pushCertificateTasks...). - Parallel(false, refreshConfigTasks...). Func("Save meta", func(_ context.Context) error { return m.specManager.SaveMeta(name, metadata) - }) + }). + Parallel(false, pushCertificateTasks...). + Parallel(false, refreshConfigTasks...) if !skipRestart { tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) @@ -844,5 +843,3 @@ func buildTLSTask( return builder.Build(), nil } - - diff --git a/pkg/cluster/manager/cacert.go b/pkg/cluster/manager/cacert.go index e2d2e466ea..56177cbae1 100644 --- a/pkg/cluster/manager/cacert.go +++ b/pkg/cluster/manager/cacert.go @@ -128,7 +128,6 @@ func (m *Manager) genAndSaveCertificate(clusterName string, globalOptions *spec. // checkCertificate check if the certificate file exists // no need to determine whether to enable tls func (m *Manager) checkCertificate(clusterName string) error { - tlsFiles := []string{ m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSCACert), m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSClientKey), @@ -141,6 +140,5 @@ func (m *Manager) checkCertificate(clusterName string) error { return perrs.Errorf("TLS file: %s does not exist", file) } } - return nil } diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index 37f93659ed..56dfee9a20 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -191,6 +191,9 @@ func (m *Manager) Deploy( // generate CA and client cert for TLS enabled cluster ca, err := m.genAndSaveCertificate(name, globalOptions) + if err != nil { + return err + } var iterErr error // error when itering over instances iterErr = nil diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index dfe78b4cfc..29269750c0 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -18,6 +18,7 @@ import ( "fmt" "os" + "github.com/fatih/color" "github.com/joomcode/errorx" perrs "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/clusterutil" @@ -69,6 +70,15 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c } } + if err := tui.PromptForConfirmOrAbortError( + fmt.Sprintf("Enable/Disable TLS %s will restart the cluster, the policy is %s.\nDo you want to continue? [y/N]:", + color.HiYellowString(name), + color.HiRedString(fmt.Sprintf("%v", !skipRestart)), + ), + ); err != nil { + return err + } + var ( sshProxyProps *tui.SSHConnectionProps = &tui.SSHConnectionProps{} ) @@ -96,6 +106,10 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c return perrs.Trace(err) } + if err := m.specManager.SaveMeta(name, metadata); err != nil { + return err + } + if !enable { // the cleanCertificate parameter will only take effect when enable is false if cleanCertificate { @@ -109,6 +123,10 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c log.Infof("Disable cluster `%s` TLS between TiDB components successfully", name) } + if skipRestart { + log.Infof("Please restart the cluster `%s` to apply the TLS configuration", name) + } + return nil } @@ -116,7 +134,6 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c // certificate file exists and reload is true // will reload certificate file func loadCertificate(m *Manager, clusterName string, globalOptions *spec.GlobalOptions, reload bool) error { - err := m.checkCertificate(clusterName) // no need to reload and the file already exists diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index 6d1cf59a13..c2739b3eb6 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -22,6 +22,7 @@ import ( "time" "github.com/creasty/defaults" + "github.com/joomcode/errorx" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/executor" @@ -29,6 +30,7 @@ import ( "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/proxy" + "github.com/pingcap/tiup/pkg/tui" clientv3 "go.etcd.io/etcd/client/v3" "golang.org/x/mod/semver" ) @@ -215,7 +217,13 @@ func (s *Specification) TLSConfig(dir string) (*tls.Config, error) { if !s.GlobalOptions.TLSEnabled { return nil, nil } - return LoadClientCert(dir) + tlsConfig, err := LoadClientCert(dir) + if err != nil { + return nil, errorx.EnsureStackTrace(err). + WithProperty(tui.SuggestionFromString("TLS is enabled, but the TLS configuration cannot be obtained")) + } + + return tlsConfig, nil } // Type implements Topology interface. From c727d524c2dd1e064151b490071fa9d23d6a6f86 Mon Sep 17 00:00:00 2001 From: srstack Date: Fri, 26 Nov 2021 20:26:32 +0800 Subject: [PATCH 04/28] Complete the enable tls operation --- pkg/cluster/manager/basic.go | 42 ++++ pkg/cluster/manager/builder.go | 341 +++++++++++++++++--------------- pkg/cluster/manager/deploy.go | 166 +++++++--------- pkg/cluster/manager/destroy.go | 4 +- pkg/cluster/manager/reload.go | 36 ++-- pkg/cluster/manager/scale_in.go | 3 +- 6 files changed, 308 insertions(+), 284 deletions(-) diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index 7ad95da9b2..f34df5033e 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" ) @@ -238,3 +239,44 @@ func (m *Manager) RestartCluster(name string, gOpt operator.Options, skipConfirm log.Infof("Restarted cluster `%s` successfully", name) return nil } + +// // monitor +// uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch +// noAgentHosts := set.NewStringSet() +// topo.IterInstance(func(inst spec.Instance) { +// // add the instance to ignore list if it marks itself as ignore_exporter +// if inst.IgnoreMonitorAgent() { +// noAgentHosts.Insert(inst.GetHost()) +// } + +// if _, found := uniqueHosts[inst.GetHost()]; !found { +// uniqueHosts[inst.GetHost()] = hostInfo{ +// ssh: inst.GetSSHPort(), +// os: inst.OS(), +// arch: inst.Arch(), +// } +// } +// }) + +// getMonitorHosts get the instance to ignore list if it marks itself as ignore_exporter +func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { + // monitor + uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch + noAgentHosts := set.NewStringSet() + topo.IterInstance(func(inst spec.Instance) { + // add the instance to ignore list if it marks itself as ignore_exporter + if inst.IgnoreMonitorAgent() { + noAgentHosts.Insert(inst.GetHost()) + } + + if _, found := uniqueHosts[inst.GetHost()]; !found { + uniqueHosts[inst.GetHost()] = hostInfo{ + ssh: inst.GetSSHPort(), + os: inst.OS(), + arch: inst.Arch(), + } + } + }) + + return uniqueHosts, noAgentHosts +} diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index d59276a70b..a1d807ecdb 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -68,10 +68,9 @@ func buildScaleOutTask( final func(b *task.Builder, name string, meta spec.Metadata, gOpt operator.Options), ) (task.Task, error) { var ( - envInitTasks []task.Task // tasks which are used to initialize environment - downloadCompTasks []task.Task // tasks which are used to download components - deployCompTasks []task.Task // tasks which are used to copy components to remote host - refreshConfigTasks []task.Task // tasks which are used to refresh configuration + envInitTasks []task.Task // tasks which are used to initialize environment + downloadCompTasks []task.Task // tasks which are used to download components + deployCompTasks []task.Task // tasks which are used to copy components to remote host ) topo := metadata.GetTopology() @@ -216,28 +215,6 @@ func buildScaleOutTask( ) } } - // generate and transfer tls cert for instance - if topo.BaseTopo().GlobalOptions.TLSEnabled { - ca, err := crypto.ReadCA( - name, - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), - ) - if err != nil { - iterErr = err - return - } - tb = tb.TLSCert( - inst.GetHost(), - inst.ComponentName(), - inst.Role(), - inst.GetMainPort(), - ca, - meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) - } t := tb.ScaleConfig(name, base.Version, @@ -257,49 +234,14 @@ func buildScaleOutTask( return nil, iterErr } - hasImported := false - noAgentHosts := set.NewStringSet() - - mergedTopo.IterInstance(func(inst spec.Instance) { - deployDir := spec.Abs(base.User, inst.DeployDir()) - // data dir would be empty for components which don't need it - dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) - // log dir will always be with values, but might not used by the component - logDir := spec.Abs(base.User, inst.LogDir()) - - // Download and copy the latest component to remote if the cluster is imported from Ansible - tb := task.NewBuilder(gOpt.DisplayMode) - if inst.IsImported() { - switch compName := inst.ComponentName(); compName { - case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: - version := m.bindVersion(compName, base.Version) - tb.Download(compName, inst.OS(), inst.Arch(), version). - CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) - } - hasImported = true - } - - // add the instance to ignore list if it marks itself as ignore_exporter - if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) - } + certificateTasks, err := buildCertificateTasks(m, name, newPart, base, gOpt, p) + if err != nil { + return nil, err + } - // Refresh all configuration - t := tb.InitConfig(name, - base.Version, - m.specManager, - inst, - base.User, - true, // always ignore config check result in scale out - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - Cache: specManager.Path(name, spec.TempConfigPath), - }, - ).Build() - refreshConfigTasks = append(refreshConfigTasks, t) - }) + // always ignore config check result in scale out + gOpt.IgnoreConfigCheck = true + refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, mergedTopo, base, gOpt, nil) // handle dir scheme changes if hasImported { @@ -308,6 +250,8 @@ func buildScaleOutTask( } } + _, noAgentHosts := getMonitorHosts(mergedTopo) + // Deploy monitor relevant components to remote dlTasks, dpTasks, err := buildMonitoredDeployTask( m, @@ -336,7 +280,8 @@ func buildScaleOutTask( builder. Parallel(false, downloadCompTasks...). Parallel(false, envInitTasks...). - Parallel(false, deployCompTasks...) + Parallel(false, deployCompTasks...). + ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...) } if afterDeploy != nil { @@ -358,7 +303,7 @@ func buildScaleOutTask( builder.Func("Start Cluster", func(ctx context.Context) error { return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg) }). - Parallel(false, refreshConfigTasks...). + ParallelStep("+ Refresh monitor configs", false, refreshConfigTasks...). Parallel(false, buildReloadPromTasks(metadata.GetTopology(), gOpt)...) } @@ -520,7 +465,84 @@ func buildMonitoredDeployTask( return } -func buildRefreshMonitoredConfigTasks( +// buildMonitoredCertificateTask generates certificate for instance and transfers it to the server +func buildMonitoredCertificateTask( + m *Manager, + name string, + uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch + noAgentHosts set.StringSet, // hosts that do not deploy monitor agents + globalOptions *spec.GlobalOptions, + monitoredOptions *spec.MonitoredOptions, + gOpt operator.Options, + p *tui.SSHConnectionProps, +) ([]*task.StepDisplay, error) { + var certificateTasks []*task.StepDisplay + + if monitoredOptions == nil { + return certificateTasks, nil + } + + if globalOptions.TLSEnabled { + // monitoring agents + for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { + for host, info := range uniqueHosts { + // skip deploying monitoring agents if the instance is marked so + if noAgentHosts.Exist(host) { + continue + } + + deployDir := spec.Abs(globalOptions.User, monitoredOptions.DeployDir) + tlsDir := filepath.Join(deployDir, "tls") + + // Deploy component + tb := task.NewBuilder(gOpt.DisplayMode). + UserSSH( + host, + info.ssh, + globalOptions.User, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + p.Password, + p.IdentityFile, + p.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + globalOptions.SSHType, + ). + Mkdir(globalOptions.User, host, tlsDir) + + if comp == spec.ComponentBlackboxExporter { + ca, innerr := crypto.ReadCA( + name, + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), + ) + if innerr != nil { + return certificateTasks, innerr + } + tb = tb.TLSCert( + host, + spec.ComponentBlackboxExporter, + spec.ComponentBlackboxExporter, + monitoredOptions.BlackboxExporterPort, + ca, + meta.DirPaths{ + Deploy: deployDir, + Cache: m.specManager.Path(name, spec.TempConfigPath), + }) + } + + certificateTasks = append(certificateTasks, tb.BuildAsStep(fmt.Sprintf(" - Generate certificate %s -> %s", comp, host))) + } + } + } + return certificateTasks, nil +} + +func buildInitMonitoredConfigTasks( specManager *spec.SpecManager, name string, uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch @@ -592,14 +614,13 @@ func buildRefreshMonitoredConfigTasks( return tasks } -func buildRegenConfigTasks( +func buildInitConfigTasks( m *Manager, name string, topo spec.Topology, base *spec.BaseMeta, gOpt operator.Options, nodes []string, - ignoreCheck bool, ) ([]*task.StepDisplay, bool) { var tasks []*task.StepDisplay hasImported := false @@ -643,7 +664,7 @@ func buildRegenConfigTasks( m.specManager, instance, base.User, - ignoreCheck, + gOpt.IgnoreConfigCheck, meta.DirPaths{ Deploy: deployDir, Data: dataDirs, @@ -651,7 +672,7 @@ func buildRegenConfigTasks( Cache: m.specManager.Path(name, spec.TempConfigPath), }, ). - BuildAsStep(fmt.Sprintf(" - Regenerate config %s -> %s", compName, instance.ID())) + BuildAsStep(fmt.Sprintf(" - Generate config %s -> %s", compName, instance.ID())) tasks = append(tasks, t) }) @@ -709,18 +730,91 @@ func buildTLSTask( p *tui.SSHConnectionProps, skipRestart bool, ) (task.Task, error) { - var ( - pushCertificateTasks []task.Task // tasks which are used to copy certificate to remote host - refreshConfigTasks []task.Task // tasks which are used to refresh configuration - ) - topo := metadata.GetTopology() base := metadata.GetBaseMeta() - specManager := m.specManager - sshType := topo.BaseTopo().GlobalOptions.SSHType + certificateTasks, err := buildCertificateTasks(m, name, topo, base, gOpt, p) + if err != nil { + return nil, err + } - var iterErr error + refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, topo, base, gOpt, nil) + + // handle dir scheme changes + if hasImported { + if err := spec.HandleImportPathMigration(name); err != nil { + return task.NewBuilder(gOpt.DisplayMode).Build(), err + } + } + + // monitor + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + MoniterCertificateTasks, err := buildMonitoredCertificateTask( + m, + name, + uniqueHosts, + noAgentHosts, + topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt, + p, + ) + if err != nil { + return nil, err + } + + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, + name, + uniqueHosts, + noAgentHosts, + *topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt, + p, + ) + + builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) + if err != nil { + return nil, err + } + + builder. + ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). + ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). + ParallelStep("+ Copy certificate to remote host", gOpt.Force, MoniterCertificateTasks...). + ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...). + Func("Save meta", func(_ context.Context) error { + return m.specManager.SaveMeta(name, metadata) + }) + + if !skipRestart { + tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) + if err != nil { + return nil, err + } + builder.Func("Restart Cluster", func(ctx context.Context) error { + return operator.Restart(ctx, topo, gOpt, tlsCfg) + }) + } + + return builder.Build(), nil +} + +// buildCertificateTasks generates certificate for instance and transfers it to the server +func buildCertificateTasks( + m *Manager, + name string, + topo spec.Topology, + base *spec.BaseMeta, + gOpt operator.Options, + p *tui.SSHConnectionProps) ([]*task.StepDisplay, error) { + var ( + iterErr error + certificateTasks []*task.StepDisplay // tasks which are used to copy certificate to remote host + ) if topo.BaseTopo().GlobalOptions.TLSEnabled { // copy certificate to remote host @@ -744,7 +838,7 @@ func buildTLSTask( p.IdentityFilePassphrase, gOpt.SSHProxyTimeout, gOpt.SSHType, - sshType, + topo.BaseTopo().GlobalOptions.SSHType, ).Mkdir(base.User, inst.GetHost(), deployDir, tlsDir) ca, err := crypto.ReadCA( @@ -765,81 +859,10 @@ func buildTLSTask( meta.DirPaths{ Deploy: deployDir, Cache: m.specManager.Path(name, spec.TempConfigPath), - }).Build() - - pushCertificateTasks = append(pushCertificateTasks, t) - }) - } - - if iterErr != nil { - return nil, iterErr - } - - hasImported := false - topo.IterInstance(func(inst spec.Instance) { - deployDir := spec.Abs(base.User, inst.DeployDir()) - // data dir would be empty for components which don't need it - dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) - // log dir will always be with values, but might not used by the component - logDir := spec.Abs(base.User, inst.LogDir()) - - // Download and copy the latest component to remote if the cluster is imported from Ansible - tb := task.NewBuilder(gOpt.DisplayMode) - if inst.IsImported() { - switch compName := inst.ComponentName(); compName { - case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: - version := m.bindVersion(compName, base.Version) - tb.Download(compName, inst.OS(), inst.Arch(), version). - CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) - } - hasImported = true - } - - // Refresh all configuration - t := tb.InitConfig(name, - base.Version, - m.specManager, - inst, - base.User, - true, // always ignore config check result in scale out - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - Cache: specManager.Path(name, spec.TempConfigPath), - }, - ).Build() - refreshConfigTasks = append(refreshConfigTasks, t) - }) - - // handle dir scheme changes - if hasImported { - if err := spec.HandleImportPathMigration(name); err != nil { - return task.NewBuilder(gOpt.DisplayMode).Build(), err - } - } - - builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) - if err != nil { - return nil, err - } - - builder. - Func("Save meta", func(_ context.Context) error { - return m.specManager.SaveMeta(name, metadata) - }). - Parallel(false, pushCertificateTasks...). - Parallel(false, refreshConfigTasks...) - - if !skipRestart { - tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) - if err != nil { - return nil, err - } - builder.Func("Restart Cluster", func(ctx context.Context) error { - return operator.Restart(ctx, topo, gOpt, tlsCfg) + }). + BuildAsStep(fmt.Sprintf(" - Generate certificate %s -> %s", inst.ComponentName(), inst.ID())) + certificateTasks = append(certificateTasks, t) }) } - - return builder.Build(), nil + return certificateTasks, iterErr } diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index 56dfee9a20..91737affbf 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -32,7 +32,6 @@ import ( "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/logger/log" - "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -185,77 +184,72 @@ func (m *Manager) Deploy( ) // Initialize environment - uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch - noAgentHosts := set.NewStringSet() + globalOptions := base.GlobalOptions - // generate CA and client cert for TLS enabled cluster - ca, err := m.genAndSaveCertificate(name, globalOptions) - if err != nil { - return err - } + metadata.SetUser(globalOptions.User) + metadata.SetVersion(clusterVersion) var iterErr error // error when itering over instances iterErr = nil + topo.IterInstance(func(inst spec.Instance) { - if _, found := uniqueHosts[inst.GetHost()]; !found { - // check for "imported" parameter, it can not be true when deploying and scaling out - // only for tidb now, need to support dm - if inst.IsImported() && m.sysName == "tidb" { - iterErr = errors.New( - "'imported' is set to 'true' for new instance, this is only used " + - "for instances imported from tidb-ansible and make no sense when " + - "deploying new instances, please delete the line or set it to 'false' for new instances") - return // skip the host to avoid issues - } + // check for "imported" parameter, it can not be true when deploying and scaling out + // only for tidb now, need to support dm + if inst.IsImported() && m.sysName == "tidb" { + iterErr = errors.New( + "'imported' is set to 'true' for new instance, this is only used " + + "for instances imported from tidb-ansible and make no sense when " + + "deploying new instances, please delete the line or set it to 'false' for new instances") + return // skip the host to avoid issues + } + }) - // add the instance to ignore list if it marks itself as ignore_exporter - if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) - } + // generate CA and client cert for TLS enabled cluster + _, err = m.genAndSaveCertificate(name, globalOptions) + if err != nil { + return err + } - uniqueHosts[inst.GetHost()] = hostInfo{ - ssh: inst.GetSSHPort(), - os: inst.OS(), - arch: inst.Arch(), - } - var dirs []string - for _, dir := range []string{globalOptions.DeployDir, globalOptions.LogDir} { - if dir == "" { - continue - } - dirs = append(dirs, spec.Abs(globalOptions.User, dir)) - } - // the default, relative path of data dir is under deploy dir - if strings.HasPrefix(globalOptions.DataDir, "/") { - dirs = append(dirs, globalOptions.DataDir) + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + + for host, hostInfo := range uniqueHosts { + var dirs []string + for _, dir := range []string{globalOptions.DeployDir, globalOptions.LogDir} { + if dir == "" { + continue } - t := task.NewBuilder(gOpt.DisplayMode). - RootSSH( - inst.GetHost(), - inst.GetSSHPort(), - opt.User, - sshConnProps.Password, - sshConnProps.IdentityFile, - sshConnProps.IdentityFilePassphrase, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - sshProxyProps.Password, - sshProxyProps.IdentityFile, - sshProxyProps.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). - EnvInit(inst.GetHost(), globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). - Mkdir(globalOptions.User, inst.GetHost(), dirs...). - BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", inst.GetHost(), inst.GetSSHPort())) - envInitTasks = append(envInitTasks, t) + dirs = append(dirs, spec.Abs(globalOptions.User, dir)) } - }) + // the default, relative path of data dir is under deploy dir + if strings.HasPrefix(globalOptions.DataDir, "/") { + dirs = append(dirs, globalOptions.DataDir) + } + t := task.NewBuilder(gOpt.DisplayMode). + RootSSH( + host, + hostInfo.ssh, + opt.User, + sshConnProps.Password, + sshConnProps.IdentityFile, + sshConnProps.IdentityFilePassphrase, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + sshProxyProps.Password, + sshProxyProps.IdentityFile, + sshProxyProps.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + globalOptions.SSHType, + ). + EnvInit(host, globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). + Mkdir(globalOptions.User, host, dirs...). + BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", host, hostInfo.ssh)) + envInitTasks = append(envInitTasks, t) + } if iterErr != nil { return iterErr @@ -280,9 +274,7 @@ func (m *Manager) Deploy( filepath.Join(deployDir, "conf"), filepath.Join(deployDir, "scripts"), } - if globalOptions.TLSEnabled { - deployDirs = append(deployDirs, filepath.Join(deployDir, "tls")) - } + t := task.NewBuilder(gOpt.DisplayMode). UserSSH( inst.GetHost(), @@ -331,36 +323,6 @@ func (m *Manager) Deploy( } } - // generate and transfer tls cert for instance - if globalOptions.TLSEnabled { - t = t.TLSCert( - inst.GetHost(), - inst.ComponentName(), - inst.Role(), - inst.GetMainPort(), - ca, - meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) - } - - // generate configs for the component - t = t.InitConfig( - name, - clusterVersion, - m.specManager, - inst, - globalOptions.User, - opt.IgnoreConfigCheck, - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }, - ) - deployCompTasks = append(deployCompTasks, t.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", inst.ComponentName(), inst.GetHost())), ) @@ -370,6 +332,14 @@ func (m *Manager) Deploy( return iterErr } + // generates certificate for instance and transfers it to the server + certificateTasks, err := buildCertificateTasks(m, name, topo, metadata.GetBaseMeta(), gOpt, sshProxyProps) + if err != nil { + return err + } + + refreshConfigTasks, _ := buildInitConfigTasks(m, name, topo, metadata.GetBaseMeta(), gOpt, nil) + // Deploy monitor relevant components to remote dlTasks, dpTasks, err := buildMonitoredDeployTask( m, @@ -393,7 +363,9 @@ func (m *Manager) Deploy( task.NewBuilder(gOpt.DisplayMode).SSHKeyGen(m.specManager.Path(name, "ssh", "id_rsa")).Build()). ParallelStep("+ Download TiDB components", false, downloadCompTasks...). ParallelStep("+ Initialize target host environments", false, envInitTasks...). - ParallelStep("+ Copy files", false, deployCompTasks...) + ParallelStep("+ Copy files", false, deployCompTasks...). + ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). + ParallelStep("+ Init instance configs", gOpt.Force, refreshConfigTasks...) if afterDeploy != nil { afterDeploy(builder, topo, gOpt) @@ -409,8 +381,6 @@ func (m *Manager) Deploy( return err } - metadata.SetUser(globalOptions.User) - metadata.SetVersion(clusterVersion) err = m.specManager.SaveMeta(name, metadata) if err != nil { diff --git a/pkg/cluster/manager/destroy.go b/pkg/cluster/manager/destroy.go index b608097c1d..57f6d108a2 100644 --- a/pkg/cluster/manager/destroy.go +++ b/pkg/cluster/manager/destroy.go @@ -131,7 +131,9 @@ func (m *Manager) DestroyTombstone( if err != nil { return err } - regenConfigTasks, _ := buildRegenConfigTasks(m, name, topo, base, gOpt, nodes, true) + + gOpt.IgnoreConfigCheck = true + regenConfigTasks, _ := buildInitConfigTasks(m, name, topo, base, gOpt, nodes) t := b. Func("FindTomestoneNodes", func(ctx context.Context) (err error) { diff --git a/pkg/cluster/manager/reload.go b/pkg/cluster/manager/reload.go index 304252ef9b..5f7602d2a3 100644 --- a/pkg/cluster/manager/reload.go +++ b/pkg/cluster/manager/reload.go @@ -27,7 +27,6 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/logger/log" - "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" ) @@ -75,25 +74,19 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo base := metadata.GetBaseMeta() // monitor - uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch - noAgentHosts := set.NewStringSet() - topo.IterInstance(func(inst spec.Instance) { - // add the instance to ignore list if it marks itself as ignore_exporter - if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) - } + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + + // init config + refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, topo, base, gOpt, nil) - if _, found := uniqueHosts[inst.GetHost()]; !found { - uniqueHosts[inst.GetHost()] = hostInfo{ - ssh: inst.GetSSHPort(), - os: inst.OS(), - arch: inst.Arch(), - } + // handle dir scheme changes + if hasImported { + if err := spec.HandleImportPathMigration(name); err != nil { + return err } - }) + } - refreshConfigTasks, hasImported := buildRegenConfigTasks(m, name, topo, base, gOpt, nil, gOpt.IgnoreConfigCheck) - monitorConfigTasks := buildRefreshMonitoredConfigTasks( + monitorConfigTasks := buildInitMonitoredConfigTasks( m.specManager, name, uniqueHosts, @@ -106,13 +99,6 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo sshProxyProps, ) - // handle dir scheme changes - if hasImported { - if err := spec.HandleImportPathMigration(name); err != nil { - return err - } - } - b, err := m.sshTaskBuilder(name, topo, base.User, gOpt) if err != nil { return err @@ -136,7 +122,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo if err != nil { return err } - b.Func("UpgradeCluster", func(ctx context.Context) error { + b.Func("Upgrade Cluster", func(ctx context.Context) error { return operator.Upgrade(ctx, topo, gOpt, tlsCfg) }) } diff --git a/pkg/cluster/manager/scale_in.go b/pkg/cluster/manager/scale_in.go index b518ed02fd..44b4b131e8 100644 --- a/pkg/cluster/manager/scale_in.go +++ b/pkg/cluster/manager/scale_in.go @@ -93,7 +93,8 @@ func (m *Manager) ScaleIn( base := metadata.GetBaseMeta() // Regenerate configuration - regenConfigTasks, hasImported := buildRegenConfigTasks(m, name, topo, base, gOpt, nodes, true) + gOpt.IgnoreConfigCheck = true + regenConfigTasks, hasImported := buildInitConfigTasks(m, name, topo, base, gOpt, nodes) // handle dir scheme changes if hasImported { From 31408fad8b2ddc1fca656fe99632ab2c58205a08 Mon Sep 17 00:00:00 2001 From: srstack Date: Sun, 28 Nov 2021 00:24:48 +0800 Subject: [PATCH 05/28] finish enable tls, but maybe pd is error --- pkg/cluster/manager/builder.go | 266 ++++++++++++--------------------- pkg/cluster/manager/deploy.go | 36 +++-- pkg/cluster/task/builder.go | 22 +++ 3 files changed, 133 insertions(+), 191 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index a1d807ecdb..5886935dc4 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -68,9 +68,9 @@ func buildScaleOutTask( final func(b *task.Builder, name string, meta spec.Metadata, gOpt operator.Options), ) (task.Task, error) { var ( - envInitTasks []task.Task // tasks which are used to initialize environment - downloadCompTasks []task.Task // tasks which are used to download components - deployCompTasks []task.Task // tasks which are used to copy components to remote host + envInitTasks []*task.StepDisplay // tasks which are used to initialize environment + downloadCompTasks []*task.StepDisplay // tasks which are used to download components + deployCompTasks []*task.StepDisplay // tasks which are used to copy components to remote host ) topo := metadata.GetTopology() @@ -114,6 +114,7 @@ func buildScaleOutTask( dirs = append(dirs, spec.Abs(globalOptions.User, dirname)) } } + t := task.NewBuilder(gOpt.DisplayMode). RootSSH( instance.GetHost(), @@ -136,12 +137,12 @@ func buildScaleOutTask( ). EnvInit(instance.GetHost(), base.User, base.Group, opt.SkipCreateUser || globalOptions.User == opt.User). Mkdir(globalOptions.User, instance.GetHost(), dirs...). - Build() + BuildAsStep(fmt.Sprintf(" - Initialized host %s ", host)) envInitTasks = append(envInitTasks, t) }) // Download missing component - downloadCompTasks = convertStepDisplaysToTasks(buildDownloadCompTasks(base.Version, newPart, gOpt, m.bindVersion)) + downloadCompTasks = buildDownloadCompTasks(base.Version, newPart, gOpt, m.bindVersion) sshType := topo.BaseTopo().GlobalOptions.SSHType @@ -161,27 +162,8 @@ func buildScaleOutTask( filepath.Join(deployDir, "conf"), filepath.Join(deployDir, "scripts"), } - if topo.BaseTopo().GlobalOptions.TLSEnabled { - deployDirs = append(deployDirs, filepath.Join(deployDir, "tls")) - } // Deploy component - tb := task.NewBuilder(gOpt.DisplayMode). - UserSSH( - inst.GetHost(), - inst.GetSSHPort(), - base.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - sshType, - ). + tb := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, sshType). Mkdir(base.User, inst.GetHost(), deployDirs...). Mkdir(base.User, inst.GetHost(), dataDirs...). Mkdir(base.User, inst.GetHost(), logDir) @@ -216,20 +198,9 @@ func buildScaleOutTask( } } - t := tb.ScaleConfig(name, - base.Version, - m.specManager, - topo, - inst, - base.User, - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - }, - ).Build() - deployCompTasks = append(deployCompTasks, t) + deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Deploy instance %s -> %s", inst.ComponentName(), inst.ID()))) }) + if iterErr != nil { return nil, iterErr } @@ -239,6 +210,9 @@ func buildScaleOutTask( return nil, err } + // init scale out config + scaleOutConfigTasks := buildScaleConfigTasks(m, name, topo, newPart, base, gOpt, p) + // always ignore config check result in scale out gOpt.IgnoreConfigCheck = true refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, mergedTopo, base, gOpt, nil) @@ -267,8 +241,24 @@ func buildScaleOutTask( if err != nil { return nil, err } - downloadCompTasks = append(downloadCompTasks, convertStepDisplaysToTasks(dlTasks)...) - deployCompTasks = append(deployCompTasks, convertStepDisplaysToTasks(dpTasks)...) + + downloadCompTasks = append(downloadCompTasks, dlTasks...) + deployCompTasks = append(deployCompTasks, dpTasks...) + + // monitor config + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, + name, + uninitializedHosts, + noAgentHosts, + *topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt, + p, + ) + scaleOutConfigTasks = append(scaleOutConfigTasks, monitorConfigTasks...) builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) if err != nil { @@ -278,10 +268,11 @@ func buildScaleOutTask( // stage2 just start and init config if !opt.Stage2 { builder. - Parallel(false, downloadCompTasks...). - Parallel(false, envInitTasks...). - Parallel(false, deployCompTasks...). - ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...) + ParallelStep("+ Download TiDB components", gOpt.Force, downloadCompTasks...). + ParallelStep("+ Initialize target host environments", gOpt.Force, envInitTasks...). + ParallelStep("+ Deploy TiDB instance", gOpt.Force, deployCompTasks...). + ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). + ParallelStep("+ Generate scale-out config", gOpt.Force, scaleOutConfigTasks...) } if afterDeploy != nil { @@ -303,7 +294,7 @@ func buildScaleOutTask( builder.Func("Start Cluster", func(ctx context.Context) error { return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg) }). - ParallelStep("+ Refresh monitor configs", false, refreshConfigTasks...). + ParallelStep("+ Refresh configs", false, refreshConfigTasks...). Parallel(false, buildReloadPromTasks(metadata.GetTopology(), gOpt)...) } @@ -320,6 +311,47 @@ func buildScaleOutTask( return builder.Build(), nil } +// buildScaleConfigTasks generates certificate for instance and transfers it to the server +func buildScaleConfigTasks( + m *Manager, + name string, + topo spec.Topology, + newPart spec.Topology, + base *spec.BaseMeta, + gOpt operator.Options, + p *tui.SSHConnectionProps) []*task.StepDisplay { + var ( + scaleConfigTasks []*task.StepDisplay // tasks which are used to copy certificate to remote host + ) + + // copy certificate to remote host + newPart.IterInstance(func(inst spec.Instance) { + deployDir := spec.Abs(base.User, inst.DeployDir()) + // data dir would be empty for components which don't need it + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(base.User, inst.LogDir()) + + t := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + ScaleConfig( + name, + base.Version, + m.specManager, + topo, + inst, + base.User, + meta.DirPaths{ + Deploy: deployDir, + Data: dataDirs, + Log: logDir, + }, + ).BuildAsStep(fmt.Sprintf(" - Generate Scale Config %s -> %s", inst.ComponentName(), inst.ID())) + scaleConfigTasks = append(scaleConfigTasks, t) + }) + + return scaleConfigTasks +} + type hostInfo struct { ssh int // ssh port of host os string // operating system @@ -327,15 +359,6 @@ type hostInfo struct { // vendor string } -// Deprecated -func convertStepDisplaysToTasks(t []*task.StepDisplay) []task.Task { - tasks := make([]task.Task, 0, len(t)) - for _, sd := range t { - tasks = append(tasks, sd) - } - return tasks -} - func buildMonitoredDeployTask( m *Manager, name string, @@ -355,7 +378,6 @@ func buildMonitoredDeployTask( // monitoring agents for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { version := m.bindVersion(comp, version) - for host, info := range uniqueHosts { // skip deploying monitoring agents if the instance is marked so if noAgentHosts.Exist(host) { @@ -394,23 +416,7 @@ func buildMonitoredDeployTask( } // Deploy component - tb := task.NewBuilder(gOpt.DisplayMode). - UserSSH( - host, - info.ssh, - globalOptions.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). + tb := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). Mkdir(globalOptions.User, host, deployDirs...). CopyComponent( comp, @@ -420,53 +426,15 @@ func buildMonitoredDeployTask( "", host, deployDir, - ). - MonitoredConfig( - name, - comp, - host, - globalOptions.ResourceControl, - monitoredOptions, - globalOptions.User, - globalOptions.TLSEnabled, - meta.DirPaths{ - Deploy: deployDir, - Data: []string{dataDir}, - Log: logDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }, ) - - if globalOptions.TLSEnabled && comp == spec.ComponentBlackboxExporter { - ca, innerr := crypto.ReadCA( - name, - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), - ) - if innerr != nil { - err = innerr - return - } - tb = tb.TLSCert( - host, - spec.ComponentBlackboxExporter, - spec.ComponentBlackboxExporter, - monitoredOptions.BlackboxExporterPort, - ca, - meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) - } - deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", comp, host))) } } return } -// buildMonitoredCertificateTask generates certificate for instance and transfers it to the server -func buildMonitoredCertificateTask( +// buildMonitoredCertificateTasks generates certificate for instance and transfers it to the server +func buildMonitoredCertificateTasks( m *Manager, name string, uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch @@ -495,23 +463,7 @@ func buildMonitoredCertificateTask( tlsDir := filepath.Join(deployDir, "tls") // Deploy component - tb := task.NewBuilder(gOpt.DisplayMode). - UserSSH( - host, - info.ssh, - globalOptions.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). + tb := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). Mkdir(globalOptions.User, host, tlsDir) if comp == spec.ComponentBlackboxExporter { @@ -575,23 +527,7 @@ func buildInitMonitoredConfigTasks( // log dir will always be with values, but might not used by the component logDir := spec.Abs(globalOptions.User, monitoredOptions.LogDir) // Generate configs - t := task.NewBuilder(gOpt.DisplayMode). - UserSSH( - host, - info.ssh, - globalOptions.User, - sshTimeout, - exeTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). + t := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). MonitoredConfig( name, comp, @@ -607,7 +543,7 @@ func buildInitMonitoredConfigTasks( Cache: specManager.Path(name, spec.TempConfigPath), }, ). - BuildAsStep(fmt.Sprintf(" - Refresh config %s -> %s", comp, host)) + BuildAsStep(fmt.Sprintf(" - Init config %s -> %s", comp, host)) tasks = append(tasks, t) } } @@ -749,7 +685,7 @@ func buildTLSTask( // monitor uniqueHosts, noAgentHosts := getMonitorHosts(topo) - MoniterCertificateTasks, err := buildMonitoredCertificateTask( + MoniterCertificateTasks, err := buildMonitoredCertificateTasks( m, name, uniqueHosts, @@ -781,14 +717,11 @@ func buildTLSTask( return nil, err } - builder. + defer builder. ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). - ParallelStep("+ Copy certificate to remote host", gOpt.Force, MoniterCertificateTasks...). - ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...). - Func("Save meta", func(_ context.Context) error { - return m.specManager.SaveMeta(name, metadata) - }) + ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, MoniterCertificateTasks...). + ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...) if !skipRestart { tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) @@ -800,6 +733,11 @@ func buildTLSTask( }) } + // finally save metadata + builder.Func("Save meta", func(_ context.Context) error { + return m.specManager.SaveMeta(name, metadata) + }) + return builder.Build(), nil } @@ -822,24 +760,8 @@ func buildCertificateTasks( deployDir := spec.Abs(base.User, inst.DeployDir()) tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) - // generate and transfer tls cert for instance - tb := task.NewBuilder(gOpt.DisplayMode). - UserSSH( - inst.GetHost(), - inst.GetSSHPort(), - base.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - topo.BaseTopo().GlobalOptions.SSHType, - ).Mkdir(base.User, inst.GetHost(), deployDir, tlsDir) + tb := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + Mkdir(base.User, inst.GetHost(), deployDir, tlsDir) ca, err := crypto.ReadCA( name, diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index 91737affbf..ac30650a3b 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -275,23 +275,7 @@ func (m *Manager) Deploy( filepath.Join(deployDir, "scripts"), } - t := task.NewBuilder(gOpt.DisplayMode). - UserSSH( - inst.GetHost(), - inst.GetSSHPort(), - globalOptions.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - sshProxyProps.Password, - sshProxyProps.IdentityFile, - sshProxyProps.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). + t := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), globalOptions.User, gOpt, sshProxyProps, globalOptions.SSHType). Mkdir(globalOptions.User, inst.GetHost(), deployDirs...). Mkdir(globalOptions.User, inst.GetHost(), dataDirs...) @@ -358,14 +342,28 @@ func (m *Manager) Deploy( downloadCompTasks = append(downloadCompTasks, dlTasks...) deployCompTasks = append(deployCompTasks, dpTasks...) + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, + name, + uniqueHosts, + noAgentHosts, + *topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt, + sshProxyProps, + ) + builder := task.NewBuilder(gOpt.DisplayMode). Step("+ Generate SSH keys", task.NewBuilder(gOpt.DisplayMode).SSHKeyGen(m.specManager.Path(name, "ssh", "id_rsa")).Build()). ParallelStep("+ Download TiDB components", false, downloadCompTasks...). ParallelStep("+ Initialize target host environments", false, envInitTasks...). - ParallelStep("+ Copy files", false, deployCompTasks...). + ParallelStep("+ Deploy TiDB instance", false, deployCompTasks...). ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). - ParallelStep("+ Init instance configs", gOpt.Force, refreshConfigTasks...) + ParallelStep("+ Init instance configs", gOpt.Force, refreshConfigTasks...). + ParallelStep("+ Init monitor configs", gOpt.Force, monitorConfigTasks...) if afterDeploy != nil { afterDeploy(builder, topo, gOpt) diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index db7cae12f9..dac8e231f5 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/proxy" + "github.com/pingcap/tiup/pkg/tui" ) // Builder is used to build TiUP task @@ -79,6 +80,27 @@ func (b *Builder) RootSSH( return b } +// NewSimpleUerSSH append a UserSSH task to the current task collection with operator.Options and SSHConnectionProps +func NewSimpleUerSSH(host string, port int, user string, gOpt operator.Options, p *tui.SSHConnectionProps, sshType executor.SSHType) *Builder { + return NewBuilder(gOpt.DisplayMode). + UserSSH( + host, + port, + user, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + p.Password, + p.IdentityFile, + p.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + sshType, + ) +} + // UserSSH append a UserSSH task to the current task collection func (b *Builder) UserSSH( host string, port int, deployUser string, sshTimeout, exeTimeout uint64, From 8c7520b6c611d90212a2dcd41cee192f0c0cc0db Mon Sep 17 00:00:00 2001 From: srstack Date: Sun, 28 Nov 2021 18:49:37 +0800 Subject: [PATCH 06/28] finish disable tls --- components/dm/spec/logic.go | 22 +++++++ pkg/cluster/manager/builder.go | 2 +- pkg/cluster/spec/alertmanager.go | 10 +++ pkg/cluster/spec/cdc.go | 9 +++ pkg/cluster/spec/drainer.go | 47 ++++++++++---- pkg/cluster/spec/grafana.go | 11 ++++ pkg/cluster/spec/instance.go | 12 ++++ pkg/cluster/spec/monitoring.go | 11 ++++ pkg/cluster/spec/pd.go | 48 ++++++++++---- pkg/cluster/spec/pump.go | 39 +++++++++--- pkg/cluster/spec/spec_test.go | 12 ++-- pkg/cluster/spec/tidb.go | 47 ++++++++++---- pkg/cluster/spec/tiflash.go | 105 ++++++++++++++++++++++--------- pkg/cluster/spec/tikv.go | 46 ++++++++++---- pkg/cluster/spec/tispark.go | 12 ++++ 15 files changed, 347 insertions(+), 86 deletions(-) diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go index f3d7d50c12..4725cc3207 100644 --- a/components/dm/spec/logic.go +++ b/components/dm/spec/logic.go @@ -142,10 +142,21 @@ func (i *MasterInstance) InitConfig( return err } + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + specConfig := spec.Config return i.MergeServerConfig(ctx, e, i.topo.ServerConfigs.Master, specConfig, paths) } +// setTLSConfig set TLS Config to support enable/disable TLS +// MasterInstance no need to configure TLS +func (i *MasterInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // ScaleConfig deploy temporary config on scaling func (i *MasterInstance) ScaleConfig( ctx context.Context, @@ -274,10 +285,21 @@ func (i *WorkerInstance) InitConfig( return err } + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + specConfig := spec.Config return i.MergeServerConfig(ctx, e, i.topo.ServerConfigs.Worker, specConfig, paths) } +// setTLSConfig set TLS Config to support enable/disable TLS +// workrsInstance no need to configure TLS +func (i *WorkerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // ScaleConfig deploy temporary config on scaling func (i *WorkerInstance) ScaleConfig( ctx context.Context, diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 5886935dc4..d3e5c74645 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -717,7 +717,7 @@ func buildTLSTask( return nil, err } - defer builder. + builder. ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, MoniterCertificateTasks...). diff --git a/pkg/cluster/spec/alertmanager.go b/pkg/cluster/spec/alertmanager.go index a0298f211f..47b52e18c7 100644 --- a/pkg/cluster/spec/alertmanager.go +++ b/pkg/cluster/spec/alertmanager.go @@ -148,6 +148,11 @@ func (i *AlertManagerInstance) InitConfig( WithWebPort(spec.WebPort).WithClusterPort(spec.ClusterPort).WithNumaNode(spec.NumaNode). AppendEndpoints(AlertManagerEndpoints(alertmanagers, deployUser, enableTLS)) + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + fp := filepath.Join(paths.Cache, fmt.Sprintf("run_alertmanager_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { return err @@ -191,3 +196,8 @@ func (i *AlertManagerInstance) ScaleConfig( i.topo = topo return i.InitConfig(ctx, e, clusterName, clusterVersion, deployUser, paths) } + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *AlertManagerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} diff --git a/pkg/cluster/spec/cdc.go b/pkg/cluster/spec/cdc.go index 1f4dfbd764..fc3dd75c1c 100644 --- a/pkg/cluster/spec/cdc.go +++ b/pkg/cluster/spec/cdc.go @@ -178,6 +178,10 @@ func (i *CDCInstance) InitConfig( spec.TZ, ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).AppendEndpoints(topo.Endpoints(deployUser)...) + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } if len(paths.Data) != 0 { cfg = cfg.PatchByVersion(clusterVersion, paths.Data[0]) } @@ -198,3 +202,8 @@ func (i *CDCInstance) InitConfig( return i.MergeServerConfig(ctx, e, globalConfig, instanceConfig, paths) } + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *CDCInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} diff --git a/pkg/cluster/spec/drainer.go b/pkg/cluster/spec/drainer.go index 5de4049a54..542b0c62d5 100644 --- a/pkg/cluster/spec/drainer.go +++ b/pkg/cluster/spec/drainer.go @@ -153,7 +153,6 @@ func (i *DrainerInstance) InitConfig( if err := i.BaseInstance.InitConfig(ctx, e, topo.GlobalOptions, deployUser, paths); err != nil { return err } - enableTLS := topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*DrainerSpec) nodeID := i.GetHost() + ":" + strconv.Itoa(i.GetPort()) @@ -181,7 +180,8 @@ func (i *DrainerInstance) InitConfig( return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -209,28 +209,51 @@ func (i *DrainerInstance) InitConfig( } // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *DrainerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ssl-ca"] = fmt.Sprintf( + configs["security.ssl-ca"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.ssl-cert"] = fmt.Sprintf( + configs["security.ssl-cert"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.ssl-key"] = fmt.Sprintf( + configs["security.ssl-key"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ssl-ca", + "security.ssl-cert", + "security.ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } diff --git a/pkg/cluster/spec/grafana.go b/pkg/cluster/spec/grafana.go index 112f286b6c..748d470590 100644 --- a/pkg/cluster/spec/grafana.go +++ b/pkg/cluster/spec/grafana.go @@ -175,6 +175,12 @@ func (i *GrafanaInstance) InitConfig( ConfigToFile(fp); err != nil { return err } + + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + dst = filepath.Join(paths.Deploy, "conf", "grafana.ini") if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err @@ -222,6 +228,11 @@ func (i *GrafanaInstance) InitConfig( return i.TransferLocalConfigFile(ctx, e, fp, dst) } +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *GrafanaInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + func (i *GrafanaInstance) initDashboards(ctx context.Context, e ctxt.Executor, spec *GrafanaSpec, paths meta.DirPaths, clusterName string) error { dashboardsDir := filepath.Join(paths.Deploy, "dashboards") if spec.DashboardDir != "" { diff --git a/pkg/cluster/spec/instance.go b/pkg/cluster/spec/instance.go index d1dadfb07c..2aed6fb45e 100644 --- a/pkg/cluster/spec/instance.go +++ b/pkg/cluster/spec/instance.go @@ -102,6 +102,7 @@ type Instance interface { Arch() string IsPatched() bool SetPatched(bool) + setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) } // PortStarted wait until a port is being listened @@ -190,9 +191,20 @@ func (i *BaseInstance) InitConfig(ctx context.Context, e ctxt.Executor, opt Glob return errors.Annotatef(err, "execute: %s", cmd) } + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + return nil } +// setTLSConfig set TLS Config to support enable/disable TLS +// baseInstance no need to configure TLS +func (i *BaseInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // TransferLocalConfigFile scp local config file to remote // Precondition: the user on remote have permission to access & mkdir of dest files func (i *BaseInstance) TransferLocalConfigFile(ctx context.Context, e ctxt.Executor, local, remote string) error { diff --git a/pkg/cluster/spec/monitoring.go b/pkg/cluster/spec/monitoring.go index 38d77430f4..f7063ae0e0 100644 --- a/pkg/cluster/spec/monitoring.go +++ b/pkg/cluster/spec/monitoring.go @@ -292,6 +292,12 @@ func (i *MonitorInstance) InitConfig( if err != nil { return err } + + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + cfig.SetRemoteConfig(string(remoteCfg)) for _, alertmanager := range spec.ExternalAlertmanagers { @@ -344,6 +350,11 @@ func (i *MonitorInstance) InitConfig( return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".yml", paths, nil) } +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *MonitorInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // We only really installRules for dm cluster because the rules(*.rules.yml) packed with the prometheus // component is designed for tidb cluster (the dm cluster use the same prometheus component with tidb // cluster), and the rules for dm cluster is packed in the dm-master component. So if deploying tidb diff --git a/pkg/cluster/spec/pd.go b/pkg/cluster/spec/pd.go index 0440a9742a..e913dbb093 100644 --- a/pkg/cluster/spec/pd.go +++ b/pkg/cluster/spec/pd.go @@ -194,7 +194,8 @@ func (i *PDInstance) InitConfig( if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -221,31 +222,55 @@ func (i *PDInstance) InitConfig( } } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *PDInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { // set TLS configs if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.cacert-path"] = fmt.Sprintf( + configs["security.cacert-path"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cert-path"] = fmt.Sprintf( + configs["security.cert-path"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.key-path"] = fmt.Sprintf( + configs["security.key-path"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.cacert-path", + "security.cert-path", + "security.key-path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } // ScaleConfig deploy temporary config on scaling @@ -265,7 +290,6 @@ func (i *PDInstance) ScaleConfig( } cluster := mustBeClusterTopo(topo) - spec := i.InstanceSpec.(*PDSpec) cfg0 := scripts.NewPDScript( i.Name, diff --git a/pkg/cluster/spec/pump.go b/pkg/cluster/spec/pump.go index 93b95fe1b1..bbc461847e 100644 --- a/pkg/cluster/spec/pump.go +++ b/pkg/cluster/spec/pump.go @@ -176,7 +176,8 @@ func (i *PumpInstance) InitConfig( return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -203,25 +204,49 @@ func (i *PumpInstance) InitConfig( } } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + return i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *PumpInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { // set TLS configs if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ssl-ca"] = fmt.Sprintf( + configs["security.ssl-ca"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.ssl-cert"] = fmt.Sprintf( + configs["security.ssl-cert"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.ssl-key"] = fmt.Sprintf( + configs["security.ssl-key"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ssl-ca", + "security.ssl-cert", + "security.ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - return i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths) + return configs, nil } diff --git a/pkg/cluster/spec/spec_test.go b/pkg/cluster/spec/spec_test.go index 335a9c7b12..ebe915f874 100644 --- a/pkg/cluster/spec/spec_test.go +++ b/pkg/cluster/spec/spec_test.go @@ -15,6 +15,7 @@ package spec import ( "bytes" + "context" "strings" "testing" @@ -585,6 +586,7 @@ pd_servers: } func (s *metaSuiteTopo) TestTiFlashStorageSection(c *C) { + ctx := context.Background() spec := &Specification{} err := yaml.Unmarshal([]byte(` tiflash_servers: @@ -605,7 +607,7 @@ tiflash_servers: // This should be the same with tiflash_server instance's "data_dir" dataDir := "/hdd0/tiflash,/hdd1/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.(*TiFlashInstance).initTiFlashConfig(cfg, "v4.0.8", spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.(*TiFlashInstance).initTiFlashConfig(ctx, cfg, "v4.0.8", spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) path, ok := conf["path"] @@ -617,7 +619,7 @@ tiflash_servers: ins := instances[0].(*TiFlashInstance) dataDir := "/ssd0/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.initTiFlashConfig(ctx, cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) _, ok := conf["path"] @@ -768,6 +770,8 @@ tiflash_servers: `), spec) c.Assert(err, IsNil) + ctx := context.Background() + flashComp := FindComponent(spec, ComponentTiFlash) instances := flashComp.Instances() c.Assert(len(instances), Equals, 1) @@ -777,7 +781,7 @@ tiflash_servers: ins := instances[0].(*TiFlashInstance) dataDir := "/ssd0/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.initTiFlashConfig(ctx, cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) // We need an empty string for 'users.default.password' for backward compatibility. Or the TiFlash process will fail to start with older versions @@ -800,7 +804,7 @@ tiflash_servers: ins := instances[0].(*TiFlashInstance) dataDir := "/ssd0/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.initTiFlashConfig(ctx, cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) // Those deprecated settings are ignored in newer versions diff --git a/pkg/cluster/spec/tidb.go b/pkg/cluster/spec/tidb.go index 4e81e6ecc1..a0b147352c 100644 --- a/pkg/cluster/spec/tidb.go +++ b/pkg/cluster/spec/tidb.go @@ -154,7 +154,8 @@ func (i *TiDBInstance) InitConfig( if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -181,31 +182,55 @@ func (i *TiDBInstance) InitConfig( } } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *TiDBInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { // set TLS configs if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.cluster-ssl-ca"] = fmt.Sprintf( + configs["security.cluster-ssl-ca"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cluster-ssl-cert"] = fmt.Sprintf( + configs["security.cluster-ssl-cert"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.cluster-ssl-key"] = fmt.Sprintf( + configs["security.cluster-ssl-key"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.cluster-ssl-ca", + "security.cluster-ssl-cert", + "security.cluster-ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } // ScaleConfig deploy temporary config on scaling diff --git a/pkg/cluster/spec/tiflash.go b/pkg/cluster/spec/tiflash.go index 3aa90869a0..c550d996f1 100644 --- a/pkg/cluster/spec/tiflash.go +++ b/pkg/cluster/spec/tiflash.go @@ -370,7 +370,7 @@ func checkTiFlashStorageConfigWithVersion(clusterVersion string, config map[stri } // InitTiFlashConfig initializes TiFlash config file with the configurations in server_configs -func (i *TiFlashInstance) initTiFlashConfig(cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { +func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { var ( pathConfig string isStorageDirsDefined bool @@ -415,26 +415,15 @@ func (i *TiFlashInstance) initTiFlashConfig(cfg *scripts.TiFlashScript, clusterV spec := i.InstanceSpec.(*TiFlashSpec) port := "http_port" - // set TLS configs enableTLS := i.topo.(*Specification).GlobalOptions.TLSEnabled + if enableTLS { port = "https_port" - if spec.Config == nil { - spec.Config = make(map[string]interface{}) - } - spec.Config["security.ca_path"] = fmt.Sprintf( - "%s/tls/%s", - paths.Deploy, - TLSCACert, - ) - spec.Config["security.cert_path"] = fmt.Sprintf( - "%s/tls/%s.crt", - paths.Deploy, - i.Role()) - spec.Config["security.key_path"] = fmt.Sprintf( - "%s/tls/%s.pem", - paths.Deploy, - i.Role()) + } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return nil, err } topo := Specification{} @@ -496,7 +485,7 @@ func (i *TiFlashInstance) mergeTiFlashInstanceConfig(clusterVersion string, glob } // InitTiFlashLearnerConfig initializes TiFlash learner config file -func (i *TiFlashInstance) InitTiFlashLearnerConfig(cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { +func (i *TiFlashInstance) InitTiFlashLearnerConfig(ctx context.Context, cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { topo := Specification{} var statusAddr string @@ -533,27 +522,87 @@ server_configs: enableTLS := i.topo.(*Specification).GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*TiFlashSpec) // set TLS configs + spec.Config, err = i.setTLSConfigWithTiFlashLearner(ctx, enableTLS, spec.Config, paths) + if err != nil { + return nil, err + } + + conf := MergeConfig(topo.ServerConfigs.TiFlashLearner, spec.Config, src) + return conf, nil +} + +// setTLSConfigWithTiFlashLearner set TLS Config to support enable/disable TLS +func (i *TiFlashInstance) setTLSConfigWithTiFlashLearner(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ca-path"] = fmt.Sprintf( + configs["security.ca-path"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cert-path"] = fmt.Sprintf( + configs["security.cert-path"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.key-path"] = fmt.Sprintf( + configs["security.key-path"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ca-path", + "security.cert-path", + "security.key-path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - conf := MergeConfig(topo.ServerConfigs.TiFlashLearner, spec.Config, src) - return conf, nil + return configs, nil +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *TiFlashInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + if enableTLS { + if configs == nil { + configs = make(map[string]interface{}) + } + configs["security.ca_path"] = fmt.Sprintf( + "%s/tls/%s", + paths.Deploy, + TLSCACert, + ) + configs["security.cert_path"] = fmt.Sprintf( + "%s/tls/%s.crt", + paths.Deploy, + i.Role()) + configs["security.key_path"] = fmt.Sprintf( + "%s/tls/%s.pem", + paths.Deploy, + i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ca_path", + "security.cert_path", + "security.key_path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } + } + + return configs, nil } // InitConfig implement Instance interface @@ -611,7 +660,7 @@ func (i *TiFlashInstance) InitConfig( return err } - conf, err := i.InitTiFlashLearnerConfig(cfg, clusterVersion, topo.ServerConfigs.TiFlashLearner, paths) + conf, err := i.InitTiFlashLearnerConfig(ctx, cfg, clusterVersion, topo.ServerConfigs.TiFlashLearner, paths) if err != nil { return err } @@ -644,7 +693,7 @@ func (i *TiFlashInstance) InitConfig( } // Init the configuration using cfg and server_configs - if conf, err = i.initTiFlashConfig(cfg, clusterVersion, topo.ServerConfigs.TiFlash, paths); err != nil { + if conf, err = i.initTiFlashConfig(ctx, cfg, clusterVersion, topo.ServerConfigs.TiFlash, paths); err != nil { return err } diff --git a/pkg/cluster/spec/tikv.go b/pkg/cluster/spec/tikv.go index de4b829953..54389e1c8b 100644 --- a/pkg/cluster/spec/tikv.go +++ b/pkg/cluster/spec/tikv.go @@ -230,7 +230,8 @@ func (i *TiKVInstance) InitConfig( return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -258,30 +259,53 @@ func (i *TiKVInstance) InitConfig( } // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *TiKVInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ca-path"] = fmt.Sprintf( + configs["security.ca-path"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cert-path"] = fmt.Sprintf( + configs["security.cert-path"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.key-path"] = fmt.Sprintf( + configs["security.key-path"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ca-path", + "security.cert-path", + "security.key-path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } // ScaleConfig deploy temporary config on scaling diff --git a/pkg/cluster/spec/tispark.go b/pkg/cluster/spec/tispark.go index 09bcdf0f24..b222088888 100644 --- a/pkg/cluster/spec/tispark.go +++ b/pkg/cluster/spec/tispark.go @@ -406,6 +406,12 @@ func (i *TiSparkWorkerInstance) InitConfig( cfg := config.NewTiSparkConfig(pdList).WithMasters(strings.Join(masterList, ",")). WithCustomFields(topo.TiSparkMasters[0].SparkConfigs) + + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + // transfer spark-defaults.conf fp := filepath.Join(paths.Cache, fmt.Sprintf("spark-defaults-%s-%d.conf", host, port)) if err := cfg.ConfigToFile(fp); err != nil { @@ -460,6 +466,12 @@ func (i *TiSparkWorkerInstance) InitConfig( return e.Transfer(ctx, fp, dst, false, 0, false) } +// setTLSConfig set TLS Config to support enable/disable TLS +// TiSparkWorkerInstance no need to configure TLS +func (i *TiSparkWorkerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // ScaleConfig deploy temporary config on scaling func (i *TiSparkWorkerInstance) ScaleConfig( ctx context.Context, From e4aba12c9d5be450f45e8350a8f897fa1d62e238 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 29 Nov 2021 00:40:47 +0800 Subject: [PATCH 07/28] add cleanup remote host tls file --- components/cluster/command/tls.go | 12 +++++ pkg/cluster/manager/basic.go | 80 ++++++++++++++++++++++++------- pkg/cluster/manager/builder.go | 14 ++++-- pkg/cluster/manager/cleanup.go | 50 +------------------ pkg/cluster/manager/tls.go | 2 +- pkg/cluster/operation/destroy.go | 2 +- pkg/cluster/spec/instance.go | 5 ++ pkg/cluster/task/tls.go | 6 +-- 8 files changed, 96 insertions(+), 75 deletions(-) diff --git a/components/cluster/command/tls.go b/components/cluster/command/tls.go index 11eae84f37..360472a872 100644 --- a/components/cluster/command/tls.go +++ b/components/cluster/command/tls.go @@ -52,6 +52,18 @@ func newTLSCmd() *cobra.Command { return perrs.New("enable or disable must be specified at least one") } + if !enableTLS && skipRestart && cleanCertificate { + return perrs.New("disabling TLS and cleaning up TLS files without restart the cluster is not allowed") + } + + if enableTLS && cleanCertificate { + return perrs.New("clean-certificate only works when tls disable") + } + + if !enableTLS && reloadCertificate { + return perrs.New("reload-certificate only works when tls enable") + } + return cm.TLS(clusterName, gOpt, enableTLS, skipRestart, cleanCertificate, reloadCertificate) }, } diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index f34df5033e..a2869a3975 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -17,6 +17,8 @@ import ( "context" "errors" "fmt" + "path" + "path/filepath" "strings" "github.com/fatih/color" @@ -240,24 +242,6 @@ func (m *Manager) RestartCluster(name string, gOpt operator.Options, skipConfirm return nil } -// // monitor -// uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch -// noAgentHosts := set.NewStringSet() -// topo.IterInstance(func(inst spec.Instance) { -// // add the instance to ignore list if it marks itself as ignore_exporter -// if inst.IgnoreMonitorAgent() { -// noAgentHosts.Insert(inst.GetHost()) -// } - -// if _, found := uniqueHosts[inst.GetHost()]; !found { -// uniqueHosts[inst.GetHost()] = hostInfo{ -// ssh: inst.GetSSHPort(), -// os: inst.OS(), -// arch: inst.Arch(), -// } -// } -// }) - // getMonitorHosts get the instance to ignore list if it marks itself as ignore_exporter func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { // monitor @@ -280,3 +264,63 @@ func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { return uniqueHosts, noAgentHosts } + +func getCleanupFile(topo spec.Topology, cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet { + // calculate file paths to be deleted before the prompt + delFileMap := make(map[string]set.StringSet) + for _, com := range topo.ComponentsByStopOrder() { + instances := com.Instances() + retainDataRoles := set.NewStringSet(retainDataRoles...) + retainDataNodes := set.NewStringSet(retainDataNodes...) + + for _, ins := range instances { + // not cleaning files of monitor agents if the instance does not have one + switch ins.ComponentName() { + case spec.ComponentNodeExporter, + spec.ComponentBlackboxExporter: + if ins.IgnoreMonitorAgent() { + continue + } + } + + // Some data of instances will be retained + dataRetained := retainDataRoles.Exist(ins.ComponentName()) || + retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) + + if dataRetained { + continue + } + + // prevent duplicate directories + dataPaths := set.NewStringSet() + logPaths := set.NewStringSet() + tlsPath := set.NewStringSet() + + if cleanupData && len(ins.DataDir()) > 0 { + for _, dataDir := range strings.Split(ins.DataDir(), ",") { + dataPaths.Insert(path.Join(dataDir, "*")) + } + } + + if cleanupLog && len(ins.LogDir()) > 0 { + for _, logDir := range strings.Split(ins.LogDir(), ",") { + logPaths.Insert(path.Join(logDir, "*.log")) + } + } + + // clean tls data + if cleanupTLS && len(ins.LogDir()) > 0 { + deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir()) + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + tlsPath.Insert(tlsDir) + } + + if delFileMap[ins.GetHost()] == nil { + delFileMap[ins.GetHost()] = set.NewStringSet() + } + delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) + } + } + + return delFileMap +} diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index d3e5c74645..17ebd82f69 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -412,7 +412,7 @@ func buildMonitoredDeployTask( filepath.Join(deployDir, "scripts"), } if globalOptions.TLSEnabled { - deployDirs = append(deployDirs, filepath.Join(deployDir, "tls")) + deployDirs = append(deployDirs, filepath.Join(deployDir, spec.TLSCertKeyDir)) } // Deploy component @@ -460,7 +460,7 @@ func buildMonitoredCertificateTasks( } deployDir := spec.Abs(globalOptions.User, monitoredOptions.DeployDir) - tlsDir := filepath.Join(deployDir, "tls") + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) // Deploy component tb := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). @@ -664,7 +664,7 @@ func buildTLSTask( metadata spec.Metadata, gOpt operator.Options, p *tui.SSHConnectionProps, - skipRestart bool, + skipRestart, cleanup bool, ) (task.Task, error) { topo := metadata.GetTopology() base := metadata.GetBaseMeta() @@ -723,6 +723,14 @@ func buildTLSTask( ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, MoniterCertificateTasks...). ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...) + // disable tls: can cleanup tls files + if !topo.BaseTopo().GlobalOptions.TLSEnabled && cleanup { + delFileMap := getCleanupFile(topo, false, false, cleanup, []string{}, []string{}) + builder.Func("CleanupCluster", func(ctx context.Context) error { + return operator.CleanupComponent(ctx, delFileMap) + }) + } + if !skipRestart { tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) if err != nil { diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index 336b91beb0..d1efa90408 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -16,8 +16,6 @@ package manager import ( "context" "fmt" - "path" - "strings" "github.com/fatih/color" "github.com/joomcode/errorx" @@ -27,7 +25,6 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/logger/log" - "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" ) @@ -49,53 +46,8 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper if err != nil { return err } - // calculate file paths to be deleted before the prompt - delFileMap := make(map[string]set.StringSet) - for _, com := range topo.ComponentsByStopOrder() { - instances := com.Instances() - retainDataRoles := set.NewStringSet(cleanOpt.RetainDataRoles...) - retainDataNodes := set.NewStringSet(cleanOpt.RetainDataNodes...) - - for _, ins := range instances { - // not cleaning files of monitor agents if the instance does not have one - switch ins.ComponentName() { - case spec.ComponentNodeExporter, - spec.ComponentBlackboxExporter: - if ins.IgnoreMonitorAgent() { - continue - } - } - - // Some data of instances will be retained - dataRetained := retainDataRoles.Exist(ins.ComponentName()) || - retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) - - if dataRetained { - continue - } - - dataPaths := set.NewStringSet() - logPaths := set.NewStringSet() - - if cleanOpt.CleanupData && len(ins.DataDir()) > 0 { - for _, dataDir := range strings.Split(ins.DataDir(), ",") { - dataPaths.Insert(path.Join(dataDir, "*")) - } - } - - if cleanOpt.CleanupLog && len(ins.LogDir()) > 0 { - for _, logDir := range strings.Split(ins.LogDir(), ",") { - logPaths.Insert(path.Join(logDir, "*.log")) - } - } - - if delFileMap[ins.GetHost()] == nil { - delFileMap[ins.GetHost()] = set.NewStringSet() - } - delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths) - } - } + delFileMap := getCleanupFile(topo, cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes) if !skipConfirm { target := "" diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 29269750c0..d88eabefbb 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -93,7 +93,7 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c // Build the tls tasks t, err := buildTLSTask( - m, name, metadata, gOpt, sshProxyProps, skipRestart) + m, name, metadata, gOpt, sshProxyProps, skipRestart, cleanCertificate) if err != nil { return err } diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index ac3eb0e8eb..5481ca42c1 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -371,7 +371,7 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T delPaths.Insert(filepath.Join(deployDir, "bin")) delPaths.Insert(filepath.Join(deployDir, "scripts")) if cls.BaseTopo().GlobalOptions.TLSEnabled { - delPaths.Insert(filepath.Join(deployDir, "tls")) + delPaths.Insert(filepath.Join(deployDir, spec.TLSCertKeyDir)) } // only delete path if it is not used by any other instance in the cluster if strings.HasPrefix(logDir, deployDir) && cls.CountDir(ins.GetHost(), logDir) == 1 { diff --git a/pkg/cluster/spec/instance.go b/pkg/cluster/spec/instance.go index 2aed6fb45e..c06a0d6f98 100644 --- a/pkg/cluster/spec/instance.go +++ b/pkg/cluster/spec/instance.go @@ -342,6 +342,11 @@ func (i *BaseInstance) DeployDir() string { return reflect.Indirect(reflect.ValueOf(i.InstanceSpec)).FieldByName("DeployDir").String() } +// TLSDir implements Instance interface +func (i *BaseInstance) TLSDir() string { + return filepath.Join(i.DeployDir()) +} + // DataDir implements Instance interface func (i *BaseInstance) DataDir() string { dataDir := reflect.Indirect(reflect.ValueOf(i.InstanceSpec)).FieldByName("DataDir") diff --git a/pkg/cluster/task/tls.go b/pkg/cluster/task/tls.go index 367a7c1636..956148e34c 100644 --- a/pkg/cluster/task/tls.go +++ b/pkg/cluster/task/tls.go @@ -102,21 +102,21 @@ func (c *TLSCert) Execute(ctx context.Context) error { return ErrNoExecutor } if err := e.Transfer(ctx, caFile, - filepath.Join(c.paths.Deploy, "tls", spec.TLSCACert), + filepath.Join(c.paths.Deploy, spec.TLSCertKeyDir, spec.TLSCACert), false, /* download */ 0, /* limit */ false /* compress */); err != nil { return errors.Annotate(err, "failed to transfer CA cert to server") } if err := e.Transfer(ctx, keyFile, - filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.pem", c.role)), + filepath.Join(c.paths.Deploy, spec.TLSCertKeyDir, fmt.Sprintf("%s.pem", c.role)), false, /* download */ 0, /* limit */ false /* compress */); err != nil { return errors.Annotate(err, "failed to transfer TLS private key to server") } if err := e.Transfer(ctx, certFile, - filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.crt", c.role)), + filepath.Join(c.paths.Deploy, spec.TLSCertKeyDir, fmt.Sprintf("%s.crt", c.role)), false, /* download */ 0, /* limit */ false /* compress */); err != nil { From 2969e37d9588972d70fc94c8635338b4f9acf9c2 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 29 Nov 2021 00:56:52 +0800 Subject: [PATCH 08/28] add clean tls file tips --- pkg/cluster/manager/builder.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 17ebd82f69..c96567ef64 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -19,6 +19,7 @@ import ( "path/filepath" "strings" + "github.com/fatih/color" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" @@ -725,7 +726,22 @@ func buildTLSTask( // disable tls: can cleanup tls files if !topo.BaseTopo().GlobalOptions.TLSEnabled && cleanup { + // get: host: set(tlsdir) delFileMap := getCleanupFile(topo, false, false, cleanup, []string{}, []string{}) + // build file list string + delFileList := "" + for host, fileList := range delFileMap { + delFileList += fmt.Sprintf("\n%s:", color.CyanString(host)) + for _, dfp := range fileList.Slice() { + delFileList += fmt.Sprintf("\n %s", dfp) + } + } + + if err := tui.PromptForConfirmOrAbortError( + "The parameter `--clean-certificate` will delete the following files: %s \nDo you want to continue? [y/N]:", delFileList); err != nil { + return nil, err + } + builder.Func("CleanupCluster", func(ctx context.Context) error { return operator.CleanupComponent(ctx, delFileMap) }) From 2ff46b9c7909ca73efe12b57b31abe159e771465 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 29 Nov 2021 01:10:28 +0800 Subject: [PATCH 09/28] add some tips --- pkg/cluster/manager/builder.go | 7 ++++--- pkg/cluster/manager/tls.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index c96567ef64..0781903f57 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/environment" + "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -729,7 +730,7 @@ func buildTLSTask( // get: host: set(tlsdir) delFileMap := getCleanupFile(topo, false, false, cleanup, []string{}, []string{}) // build file list string - delFileList := "" + delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(name, spec.TLSCertKeyDir)) for host, fileList := range delFileMap { delFileList += fmt.Sprintf("\n%s:", color.CyanString(host)) for _, dfp := range fileList.Slice() { @@ -737,8 +738,8 @@ func buildTLSTask( } } - if err := tui.PromptForConfirmOrAbortError( - "The parameter `--clean-certificate` will delete the following files: %s \nDo you want to continue? [y/N]:", delFileList); err != nil { + log.Warnf(color.YellowString("The parameter `--clean-certificate` will delete the following files: %s"), delFileList) + if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { return nil, err } diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index d88eabefbb..0749f58b71 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -71,7 +71,7 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c } if err := tui.PromptForConfirmOrAbortError( - fmt.Sprintf("Enable/Disable TLS %s will restart the cluster, the policy is %s.\nDo you want to continue? [y/N]:", + fmt.Sprintf("Enable/Disable TLS will restart the cluster `%s` , the policy is %s.\nDo you want to continue? [y/N]:", color.HiYellowString(name), color.HiRedString(fmt.Sprintf("%v", !skipRestart)), ), @@ -115,6 +115,7 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c if cleanCertificate { os.RemoveAll(m.specManager.Path(name, spec.TLSCertKeyDir)) } + log.Infof("\tCleanup localhost tls file success") } if enable { From 620bb7cf0da1de6294c83813bba6253bd45fa315 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 29 Nov 2021 15:20:07 +0800 Subject: [PATCH 10/28] fix scale-out operation not create tls files --- components/dm/spec/logic.go | 2 +- pkg/cluster/manager/basic.go | 2 +- pkg/cluster/manager/builder.go | 22 +++++++++++++++++++--- pkg/cluster/manager/cleanup.go | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go index 4725cc3207..1dca530670 100644 --- a/components/dm/spec/logic.go +++ b/components/dm/spec/logic.go @@ -347,7 +347,7 @@ func (topo *Specification) ComponentsByStartOrder() (comps []Component) { // "dm-master", "dm-worker" comps = append(comps, &DMMasterComponent{topo}) comps = append(comps, &DMWorkerComponent{topo}) - comps = append(comps, &spec.MonitorComponent{Topology: topo}) + comps = append(comps, &spec.MonitorComponent{Topology: topo}) // prometheus comps = append(comps, &spec.GrafanaComponent{Topology: topo}) comps = append(comps, &spec.AlertManagerComponent{Topology: topo}) return diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index a2869a3975..fa28c0d934 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -265,7 +265,7 @@ func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { return uniqueHosts, noAgentHosts } -func getCleanupFile(topo spec.Topology, cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet { +func getCleanupFiles(topo spec.Topology, cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet { // calculate file paths to be deleted before the prompt delFileMap := make(map[string]set.StringSet) for _, com := range topo.ComponentsByStopOrder() { diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 0781903f57..2cb1b296f7 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -247,6 +247,22 @@ func buildScaleOutTask( downloadCompTasks = append(downloadCompTasks, dlTasks...) deployCompTasks = append(deployCompTasks, dpTasks...) + // monitor tls file + moniterCertificateTasks, err := buildMonitoredCertificateTasks( + m, + name, + uninitializedHosts, + noAgentHosts, + topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt, + p, + ) + if err != nil { + return nil, err + } + certificateTasks = append(certificateTasks, moniterCertificateTasks...) + // monitor config monitorConfigTasks := buildInitMonitoredConfigTasks( m.specManager, @@ -687,7 +703,7 @@ func buildTLSTask( // monitor uniqueHosts, noAgentHosts := getMonitorHosts(topo) - MoniterCertificateTasks, err := buildMonitoredCertificateTasks( + moniterCertificateTasks, err := buildMonitoredCertificateTasks( m, name, uniqueHosts, @@ -722,13 +738,13 @@ func buildTLSTask( builder. ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). - ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, MoniterCertificateTasks...). + ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, moniterCertificateTasks...). ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...) // disable tls: can cleanup tls files if !topo.BaseTopo().GlobalOptions.TLSEnabled && cleanup { // get: host: set(tlsdir) - delFileMap := getCleanupFile(topo, false, false, cleanup, []string{}, []string{}) + delFileMap := getCleanupFiles(topo, false, false, cleanup, []string{}, []string{}) // build file list string delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(name, spec.TLSCertKeyDir)) for host, fileList := range delFileMap { diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index d1efa90408..8374923eb8 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -47,7 +47,7 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper return err } // calculate file paths to be deleted before the prompt - delFileMap := getCleanupFile(topo, cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes) + delFileMap := getCleanupFiles(topo, cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes) if !skipConfirm { target := "" From a92b58a4bd67a5fd2313300cc6f1125346d3c6c4 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 29 Nov 2021 16:05:51 +0800 Subject: [PATCH 11/28] fix deploy operation not create tls files --- pkg/cluster/manager/builder.go | 3 --- pkg/cluster/manager/deploy.go | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 2cb1b296f7..b2d5662d88 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -429,9 +429,6 @@ func buildMonitoredDeployTask( filepath.Join(deployDir, "conf"), filepath.Join(deployDir, "scripts"), } - if globalOptions.TLSEnabled { - deployDirs = append(deployDirs, filepath.Join(deployDir, spec.TLSCertKeyDir)) - } // Deploy component tb := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index ac30650a3b..c3fcbf8a75 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -342,6 +342,22 @@ func (m *Manager) Deploy( downloadCompTasks = append(downloadCompTasks, dlTasks...) deployCompTasks = append(deployCompTasks, dpTasks...) + // monitor tls file + moniterCertificateTasks, err := buildMonitoredCertificateTasks( + m, + name, + uniqueHosts, + noAgentHosts, + topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt, + sshProxyProps, + ) + if err != nil { + return err + } + certificateTasks = append(certificateTasks, moniterCertificateTasks...) + monitorConfigTasks := buildInitMonitoredConfigTasks( m.specManager, name, From 3a3dbe49d74020f53655dc3115de8c44c8837b7b Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 29 Nov 2021 17:05:40 +0800 Subject: [PATCH 12/28] fix cannot cleanup monitor data/log/tls files --- pkg/cluster/manager/basic.go | 99 ++++++++++++++++++++++++++++++++---- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index fa28c0d934..5c645b2ee5 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -265,16 +265,42 @@ func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { return uniqueHosts, noAgentHosts } +// cleanUpFiles record the file that needs to be cleaned up +type cleanUpFiles struct { + cleanupData bool // whether to clean up the data + cleanupLog bool // whether to clean up the log + cleanupTLS bool // whether to clean up the tls files + retainDataRoles []string // roles that don't clean up + retainDataNodes []string // roles that don't clean up + delFileMap map[string]set.StringSet // +} + func getCleanupFiles(topo spec.Topology, cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet { + c := &cleanUpFiles{ + cleanupData: cleanupData, + cleanupLog: cleanupLog, + cleanupTLS: cleanupTLS, + retainDataRoles: retainDataRoles, + retainDataNodes: retainDataNodes, + delFileMap: make(map[string]set.StringSet), + } + // calculate file paths to be deleted before the prompt - delFileMap := make(map[string]set.StringSet) + c.instanceCleanupFiles(topo) + c.monitorCleanupFoles(topo) + + return c.delFileMap +} + +func (c *cleanUpFiles) instanceCleanupFiles(topo spec.Topology) { for _, com := range topo.ComponentsByStopOrder() { instances := com.Instances() - retainDataRoles := set.NewStringSet(retainDataRoles...) - retainDataNodes := set.NewStringSet(retainDataNodes...) + retainDataRoles := set.NewStringSet(c.retainDataRoles...) + retainDataNodes := set.NewStringSet(c.retainDataNodes...) for _, ins := range instances { // not cleaning files of monitor agents if the instance does not have one + // may not work switch ins.ComponentName() { case spec.ComponentNodeExporter, spec.ComponentBlackboxExporter: @@ -296,31 +322,84 @@ func getCleanupFiles(topo spec.Topology, cleanupData, cleanupLog, cleanupTLS boo logPaths := set.NewStringSet() tlsPath := set.NewStringSet() - if cleanupData && len(ins.DataDir()) > 0 { + if c.cleanupData && len(ins.DataDir()) > 0 { for _, dataDir := range strings.Split(ins.DataDir(), ",") { dataPaths.Insert(path.Join(dataDir, "*")) } } - if cleanupLog && len(ins.LogDir()) > 0 { + if c.cleanupLog && len(ins.LogDir()) > 0 { for _, logDir := range strings.Split(ins.LogDir(), ",") { logPaths.Insert(path.Join(logDir, "*.log")) } } // clean tls data - if cleanupTLS && len(ins.LogDir()) > 0 { + if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir()) tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) tlsPath.Insert(tlsDir) } - if delFileMap[ins.GetHost()] == nil { - delFileMap[ins.GetHost()] = set.NewStringSet() + if c.delFileMap[ins.GetHost()] == nil { + c.delFileMap[ins.GetHost()] = set.NewStringSet() } - delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) + c.delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) } } +} + +func (c *cleanUpFiles) monitorCleanupFoles(topo spec.Topology) { + monitoredOptions := topo.BaseTopo().MonitoredOptions + if monitoredOptions == nil { + return + } + user := topo.BaseTopo().GlobalOptions.User + + // get the host with monitor installed + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + retainDataNodes := set.NewStringSet(c.retainDataNodes...) + + // monitoring agents + for host := range uniqueHosts { + // determine if host don't need to delete + dataRetained := noAgentHosts.Exist(host) || retainDataNodes.Exist(host) + if dataRetained { + continue + } - return delFileMap + deployDir := spec.Abs(user, monitoredOptions.DeployDir) + + // prevent duplicate directories + dataPaths := set.NewStringSet() + logPaths := set.NewStringSet() + tlsPath := set.NewStringSet() + + // data dir would be empty for components which don't need it + dataDir := monitoredOptions.DataDir + if c.cleanupData && len(dataDir) > 0 { + // the default data_dir is relative to deploy_dir + if !strings.HasPrefix(dataDir, "/") { + dataDir = filepath.Join(deployDir, dataDir) + } + dataPaths.Insert(path.Join(dataDir, "*")) + } + + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(user, monitoredOptions.LogDir) + if c.cleanupLog && len(logDir) > 0 { + logPaths.Insert(path.Join(logDir, "*.log")) + } + + // clean tls data + if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + tlsPath.Insert(tlsDir) + } + + if c.delFileMap[host] == nil { + c.delFileMap[host] = set.NewStringSet() + } + c.delFileMap[host].Join(logPaths).Join(dataPaths).Join(tlsPath) + } } From 3eb45286391f1742f865b7d983e57e7a757b8a5b Mon Sep 17 00:00:00 2001 From: srstack Date: Tue, 30 Nov 2021 16:17:21 +0800 Subject: [PATCH 13/28] fix --- pkg/cluster/manager/basic.go | 141 ------------------------------- pkg/cluster/manager/cleanup.go | 147 +++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 141 deletions(-) diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index 5c645b2ee5..26caf29b2d 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -17,8 +17,6 @@ import ( "context" "errors" "fmt" - "path" - "path/filepath" "strings" "github.com/fatih/color" @@ -264,142 +262,3 @@ func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { return uniqueHosts, noAgentHosts } - -// cleanUpFiles record the file that needs to be cleaned up -type cleanUpFiles struct { - cleanupData bool // whether to clean up the data - cleanupLog bool // whether to clean up the log - cleanupTLS bool // whether to clean up the tls files - retainDataRoles []string // roles that don't clean up - retainDataNodes []string // roles that don't clean up - delFileMap map[string]set.StringSet // -} - -func getCleanupFiles(topo spec.Topology, cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet { - c := &cleanUpFiles{ - cleanupData: cleanupData, - cleanupLog: cleanupLog, - cleanupTLS: cleanupTLS, - retainDataRoles: retainDataRoles, - retainDataNodes: retainDataNodes, - delFileMap: make(map[string]set.StringSet), - } - - // calculate file paths to be deleted before the prompt - c.instanceCleanupFiles(topo) - c.monitorCleanupFoles(topo) - - return c.delFileMap -} - -func (c *cleanUpFiles) instanceCleanupFiles(topo spec.Topology) { - for _, com := range topo.ComponentsByStopOrder() { - instances := com.Instances() - retainDataRoles := set.NewStringSet(c.retainDataRoles...) - retainDataNodes := set.NewStringSet(c.retainDataNodes...) - - for _, ins := range instances { - // not cleaning files of monitor agents if the instance does not have one - // may not work - switch ins.ComponentName() { - case spec.ComponentNodeExporter, - spec.ComponentBlackboxExporter: - if ins.IgnoreMonitorAgent() { - continue - } - } - - // Some data of instances will be retained - dataRetained := retainDataRoles.Exist(ins.ComponentName()) || - retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) - - if dataRetained { - continue - } - - // prevent duplicate directories - dataPaths := set.NewStringSet() - logPaths := set.NewStringSet() - tlsPath := set.NewStringSet() - - if c.cleanupData && len(ins.DataDir()) > 0 { - for _, dataDir := range strings.Split(ins.DataDir(), ",") { - dataPaths.Insert(path.Join(dataDir, "*")) - } - } - - if c.cleanupLog && len(ins.LogDir()) > 0 { - for _, logDir := range strings.Split(ins.LogDir(), ",") { - logPaths.Insert(path.Join(logDir, "*.log")) - } - } - - // clean tls data - if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { - deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir()) - tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) - tlsPath.Insert(tlsDir) - } - - if c.delFileMap[ins.GetHost()] == nil { - c.delFileMap[ins.GetHost()] = set.NewStringSet() - } - c.delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) - } - } -} - -func (c *cleanUpFiles) monitorCleanupFoles(topo spec.Topology) { - monitoredOptions := topo.BaseTopo().MonitoredOptions - if monitoredOptions == nil { - return - } - user := topo.BaseTopo().GlobalOptions.User - - // get the host with monitor installed - uniqueHosts, noAgentHosts := getMonitorHosts(topo) - retainDataNodes := set.NewStringSet(c.retainDataNodes...) - - // monitoring agents - for host := range uniqueHosts { - // determine if host don't need to delete - dataRetained := noAgentHosts.Exist(host) || retainDataNodes.Exist(host) - if dataRetained { - continue - } - - deployDir := spec.Abs(user, monitoredOptions.DeployDir) - - // prevent duplicate directories - dataPaths := set.NewStringSet() - logPaths := set.NewStringSet() - tlsPath := set.NewStringSet() - - // data dir would be empty for components which don't need it - dataDir := monitoredOptions.DataDir - if c.cleanupData && len(dataDir) > 0 { - // the default data_dir is relative to deploy_dir - if !strings.HasPrefix(dataDir, "/") { - dataDir = filepath.Join(deployDir, dataDir) - } - dataPaths.Insert(path.Join(dataDir, "*")) - } - - // log dir will always be with values, but might not used by the component - logDir := spec.Abs(user, monitoredOptions.LogDir) - if c.cleanupLog && len(logDir) > 0 { - logPaths.Insert(path.Join(logDir, "*.log")) - } - - // clean tls data - if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { - tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) - tlsPath.Insert(tlsDir) - } - - if c.delFileMap[host] == nil { - c.delFileMap[host] = set.NewStringSet() - } - c.delFileMap[host].Join(logPaths).Join(dataPaths).Join(tlsPath) - } -} diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index 8374923eb8..c6c8c2ae07 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -16,6 +16,9 @@ package manager import ( "context" "fmt" + "path" + "path/filepath" + "strings" "github.com/fatih/color" "github.com/joomcode/errorx" @@ -25,6 +28,7 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" ) @@ -107,3 +111,146 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper log.Infof("Cleanup cluster `%s` successfully", name) return nil } + +// cleanupFiles record the file that needs to be cleaned up +type cleanupFiles struct { + cleanupData bool // whether to clean up the data + cleanupLog bool // whether to clean up the log + cleanupTLS bool // whether to clean up the tls files + retainDataRoles []string // roles that don't clean up + retainDataNodes []string // roles that don't clean up + delFileMap map[string]set.StringSet +} + +// getCleanupFiles get the files that need to be deleted +func getCleanupFiles(topo spec.Topology, + cleanupData, cleanupLog, cleanupTLS bool, retainDataRoles, retainDataNodes []string) map[string]set.StringSet { + c := &cleanupFiles{ + cleanupData: cleanupData, + cleanupLog: cleanupLog, + cleanupTLS: cleanupTLS, + retainDataRoles: retainDataRoles, + retainDataNodes: retainDataNodes, + delFileMap: make(map[string]set.StringSet), + } + + // calculate file paths to be deleted before the prompt + c.instanceCleanupFiles(topo) + c.monitorCleanupFiles(topo) + + return c.delFileMap +} + +// instanceCleanupFiles get the files that need to be deleted in the component +func (c *cleanupFiles) instanceCleanupFiles(topo spec.Topology) { + for _, com := range topo.ComponentsByStopOrder() { + instances := com.Instances() + retainDataRoles := set.NewStringSet(c.retainDataRoles...) + retainDataNodes := set.NewStringSet(c.retainDataNodes...) + + for _, ins := range instances { + // not cleaning files of monitor agents if the instance does not have one + // may not work + switch ins.ComponentName() { + case spec.ComponentNodeExporter, + spec.ComponentBlackboxExporter: + if ins.IgnoreMonitorAgent() { + continue + } + } + + // Some data of instances will be retained + dataRetained := retainDataRoles.Exist(ins.ComponentName()) || + retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) + + if dataRetained { + continue + } + + // prevent duplicate directories + dataPaths := set.NewStringSet() + logPaths := set.NewStringSet() + tlsPath := set.NewStringSet() + + if c.cleanupData && len(ins.DataDir()) > 0 { + for _, dataDir := range strings.Split(ins.DataDir(), ",") { + dataPaths.Insert(path.Join(dataDir, "*")) + } + } + + if c.cleanupLog && len(ins.LogDir()) > 0 { + for _, logDir := range strings.Split(ins.LogDir(), ",") { + logPaths.Insert(path.Join(logDir, "*.log")) + } + } + + // clean tls data + if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { + deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir()) + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + tlsPath.Insert(tlsDir) + } + + if c.delFileMap[ins.GetHost()] == nil { + c.delFileMap[ins.GetHost()] = set.NewStringSet() + } + c.delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) + } + } +} + +// monitorCleanupFiles get the files that need to be deleted in the mointor +func (c *cleanupFiles) monitorCleanupFiles(topo spec.Topology) { + monitoredOptions := topo.BaseTopo().MonitoredOptions + if monitoredOptions == nil { + return + } + user := topo.BaseTopo().GlobalOptions.User + + // get the host with monitor installed + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + retainDataNodes := set.NewStringSet(c.retainDataNodes...) + + // monitoring agents + for host := range uniqueHosts { + // determine if host don't need to delete + dataRetained := noAgentHosts.Exist(host) || retainDataNodes.Exist(host) + if dataRetained { + continue + } + + deployDir := spec.Abs(user, monitoredOptions.DeployDir) + + // prevent duplicate directories + dataPaths := set.NewStringSet() + logPaths := set.NewStringSet() + tlsPath := set.NewStringSet() + + // data dir would be empty for components which don't need it + dataDir := monitoredOptions.DataDir + if c.cleanupData && len(dataDir) > 0 { + // the default data_dir is relative to deploy_dir + if !strings.HasPrefix(dataDir, "/") { + dataDir = filepath.Join(deployDir, dataDir) + } + dataPaths.Insert(path.Join(dataDir, "*")) + } + + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(user, monitoredOptions.LogDir) + if c.cleanupLog && len(logDir) > 0 { + logPaths.Insert(path.Join(logDir, "*.log")) + } + + // clean tls data + if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + tlsPath.Insert(tlsDir) + } + + if c.delFileMap[host] == nil { + c.delFileMap[host] = set.NewStringSet() + } + c.delFileMap[host].Join(logPaths).Join(dataPaths).Join(tlsPath) + } +} From b5399853accb8b5b2f478c9f128e9c03130cc58e Mon Sep 17 00:00:00 2001 From: srstack Date: Wed, 1 Dec 2021 15:57:31 +0800 Subject: [PATCH 14/28] delete skip-restart flag --- components/cluster/command/tls.go | 8 +----- pkg/cluster/manager/builder.go | 16 +++++------ pkg/cluster/manager/tls.go | 45 ++++++++++++++++--------------- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/components/cluster/command/tls.go b/components/cluster/command/tls.go index 360472a872..ceb9d2ec7d 100644 --- a/components/cluster/command/tls.go +++ b/components/cluster/command/tls.go @@ -22,7 +22,6 @@ import ( func newTLSCmd() *cobra.Command { var ( - skipRestart bool // skip restart cluster, but the tls option does not work reloadCertificate bool // reload certificate when the cluster enable encrypted communication cleanCertificate bool // cleanup certificate when the cluster disable encrypted communication enableTLS bool @@ -52,10 +51,6 @@ func newTLSCmd() *cobra.Command { return perrs.New("enable or disable must be specified at least one") } - if !enableTLS && skipRestart && cleanCertificate { - return perrs.New("disabling TLS and cleaning up TLS files without restart the cluster is not allowed") - } - if enableTLS && cleanCertificate { return perrs.New("clean-certificate only works when tls disable") } @@ -64,13 +59,12 @@ func newTLSCmd() *cobra.Command { return perrs.New("reload-certificate only works when tls enable") } - return cm.TLS(clusterName, gOpt, enableTLS, skipRestart, cleanCertificate, reloadCertificate) + return cm.TLS(clusterName, gOpt, enableTLS, cleanCertificate, reloadCertificate, skipConfirm) }, } cmd.Flags().BoolVar(&cleanCertificate, "clean-certificate", false, "Clean up the certificate file if it already exists when disable encrypted communication") cmd.Flags().BoolVar(&reloadCertificate, "reload-certificate", false, "Reload the certificate file if it already exists when enable encrypted communication") - cmd.Flags().BoolVar(&skipRestart, "skip-restart", false, "Only refresh configuration and load certificate to remote, but do not restart services") return cmd } diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index b2d5662d88..0dd38204bd 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -679,7 +679,7 @@ func buildTLSTask( metadata spec.Metadata, gOpt operator.Options, p *tui.SSHConnectionProps, - skipRestart, cleanup bool, + cleanup bool, ) (task.Task, error) { topo := metadata.GetTopology() base := metadata.GetBaseMeta() @@ -761,15 +761,13 @@ func buildTLSTask( }) } - if !skipRestart { - tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) - if err != nil { - return nil, err - } - builder.Func("Restart Cluster", func(ctx context.Context) error { - return operator.Restart(ctx, topo, gOpt, tlsCfg) - }) + tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) + if err != nil { + return nil, err } + builder.Func("Restart Cluster", func(ctx context.Context) error { + return operator.Restart(ctx, topo, gOpt, tlsCfg) + }) // finally save metadata builder.Func("Save meta", func(_ context.Context) error { diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 0749f58b71..a0844e5e94 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -32,7 +32,7 @@ import ( ) // TLS set cluster enable/disable encrypt communication by tls -func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, cleanCertificate, reloadCertificate bool) error { +func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertificate, reloadCertificate, skipConfirm bool) error { if err := clusterutil.ValidateClusterNameOrError(name); err != nil { return err } @@ -61,21 +61,7 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c } } - // check tiflash version - if clusterSpec, ok := topo.(*spec.Specification); ok { - if clusterSpec.GlobalOptions.TLSEnabled && - semver.Compare(base.Version, "v4.0.5") < 0 && - len(clusterSpec.TiFlashServers) > 0 { - return fmt.Errorf("TiFlash %s is not supported in TLS enabled cluster", base.Version) - } - } - - if err := tui.PromptForConfirmOrAbortError( - fmt.Sprintf("Enable/Disable TLS will restart the cluster `%s` , the policy is %s.\nDo you want to continue? [y/N]:", - color.HiYellowString(name), - color.HiRedString(fmt.Sprintf("%v", !skipRestart)), - ), - ); err != nil { + if err := checkTLSEnv(topo, name, base.Version, skipConfirm); err != nil { return err } @@ -93,7 +79,7 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c // Build the tls tasks t, err := buildTLSTask( - m, name, metadata, gOpt, sshProxyProps, skipRestart, cleanCertificate) + m, name, metadata, gOpt, sshProxyProps, cleanCertificate) if err != nil { return err } @@ -123,11 +109,6 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, skipRestart, c } else { log.Infof("Disable cluster `%s` TLS between TiDB components successfully", name) } - - if skipRestart { - log.Infof("Please restart the cluster `%s` to apply the TLS configuration", name) - } - return nil } @@ -146,3 +127,23 @@ func loadCertificate(m *Manager, clusterName string, globalOptions *spec.GlobalO return err } + +func checkTLSEnv(topo spec.Topology, clusterName, version string, skipConfirm bool) error { + // check tiflash version + if clusterSpec, ok := topo.(*spec.Specification); ok { + if clusterSpec.GlobalOptions.TLSEnabled && + semver.Compare(version, "v4.0.5") < 0 && + len(clusterSpec.TiFlashServers) > 0 { + return fmt.Errorf("TiFlash %s is not supported in TLS enabled cluster", version) + } + } + + if !skipConfirm { + return tui.PromptForConfirmOrAbortError( + fmt.Sprintf("Enable/Disable TLS will %s the cluster `%s`\nDo you want to continue? [y/N]:", + color.HiYellowString("restart"), + color.HiYellowString(clusterName), + )) + } + return nil +} From 0d39bc0ca9592029512ba973a59e6f02acfa842b Mon Sep 17 00:00:00 2001 From: srstack Date: Wed, 1 Dec 2021 16:55:04 +0800 Subject: [PATCH 15/28] fix skip confirm --- pkg/cluster/manager/builder.go | 24 +++--------------------- pkg/cluster/manager/tls.go | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 0dd38204bd..2119684beb 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -19,13 +19,11 @@ import ( "path/filepath" "strings" - "github.com/fatih/color" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/environment" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -679,7 +677,7 @@ func buildTLSTask( metadata spec.Metadata, gOpt operator.Options, p *tui.SSHConnectionProps, - cleanup bool, + delFileMap map[string]set.StringSet, ) (task.Task, error) { topo := metadata.GetTopology() base := metadata.GetBaseMeta() @@ -738,24 +736,8 @@ func buildTLSTask( ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, moniterCertificateTasks...). ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...) - // disable tls: can cleanup tls files - if !topo.BaseTopo().GlobalOptions.TLSEnabled && cleanup { - // get: host: set(tlsdir) - delFileMap := getCleanupFiles(topo, false, false, cleanup, []string{}, []string{}) - // build file list string - delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(name, spec.TLSCertKeyDir)) - for host, fileList := range delFileMap { - delFileList += fmt.Sprintf("\n%s:", color.CyanString(host)) - for _, dfp := range fileList.Slice() { - delFileList += fmt.Sprintf("\n %s", dfp) - } - } - - log.Warnf(color.YellowString("The parameter `--clean-certificate` will delete the following files: %s"), delFileList) - if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { - return nil, err - } - + // cleanup tls files only in tls disable + if !topo.BaseTopo().GlobalOptions.TLSEnabled { builder.Func("CleanupCluster", func(ctx context.Context) error { return operator.CleanupComponent(ctx, delFileMap) }) diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index a0844e5e94..36a8e0b1b3 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -27,6 +27,7 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" "golang.org/x/mod/semver" ) @@ -77,9 +78,30 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica } } + delFileMap := make(map[string]set.StringSet) + if !enable && cleanCertificate { + // get: host: set(tlsdir) + delFileMap := getCleanupFiles(topo, false, false, cleanCertificate, []string{}, []string{}) + // build file list string + delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(name, spec.TLSCertKeyDir)) + for host, fileList := range delFileMap { + delFileList += fmt.Sprintf("\n%s:", color.CyanString(host)) + for _, dfp := range fileList.Slice() { + delFileList += fmt.Sprintf("\n %s", dfp) + } + } + + if !skipConfirm { + log.Warnf(color.YellowString("The parameter `--clean-certificate` will delete the following files: %s"), delFileList) + if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { + return err + } + } + } + // Build the tls tasks t, err := buildTLSTask( - m, name, metadata, gOpt, sshProxyProps, cleanCertificate) + m, name, metadata, gOpt, sshProxyProps, delFileMap) if err != nil { return err } @@ -128,6 +150,7 @@ func loadCertificate(m *Manager, clusterName string, globalOptions *spec.GlobalO return err } +// checkTLSEnv check tiflash vserson and show confirm func checkTLSEnv(topo spec.Topology, clusterName, version string, skipConfirm bool) error { // check tiflash version if clusterSpec, ok := topo.(*spec.Specification); ok { From 934f431b42eeedff34e045be7d0e7aa093350fa7 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 2 Dec 2021 11:10:31 +0800 Subject: [PATCH 16/28] skip confirm --- pkg/cluster/manager/tls.go | 52 ++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 36a8e0b1b3..864007114a 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -78,25 +78,10 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica } } - delFileMap := make(map[string]set.StringSet) - if !enable && cleanCertificate { - // get: host: set(tlsdir) - delFileMap := getCleanupFiles(topo, false, false, cleanCertificate, []string{}, []string{}) - // build file list string - delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(name, spec.TLSCertKeyDir)) - for host, fileList := range delFileMap { - delFileList += fmt.Sprintf("\n%s:", color.CyanString(host)) - for _, dfp := range fileList.Slice() { - delFileList += fmt.Sprintf("\n %s", dfp) - } - } - - if !skipConfirm { - log.Warnf(color.YellowString("The parameter `--clean-certificate` will delete the following files: %s"), delFileList) - if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { - return err - } - } + // delFileMap: files that need to be cleaned up, if flag -- cleanCertificate are used + delFileMap, err := getTLSFileMap(m, name, topo, enable, cleanCertificate, skipConfirm) + if err != nil { + return err } // Build the tls tasks @@ -170,3 +155,32 @@ func checkTLSEnv(topo spec.Topology, clusterName, version string, skipConfirm bo } return nil } + +// getTLSFileMap +func getTLSFileMap(m *Manager, clusterName string, topo spec.Topology, + enableTLS, cleanCertificate, skipConfirm bool) (map[string]set.StringSet, error) { + delFileMap := make(map[string]set.StringSet) + + if !enableTLS && cleanCertificate { + // get: host: set(tlsdir) + delFileMap = getCleanupFiles(topo, false, false, cleanCertificate, []string{}, []string{}) + // build file list string + delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(clusterName, spec.TLSCertKeyDir)) + for host, fileList := range delFileMap { + delFileList += fmt.Sprintf("\n%s:", color.CyanString(host)) + for _, dfp := range fileList.Slice() { + delFileList += fmt.Sprintf("\n %s", dfp) + } + } + + log.Warnf("The parameter `%s` will delete the following files: %s", color.YellowString("--clean-certificate"), delFileList) + + if !skipConfirm { + if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { + return delFileMap, err + } + } + } + + return delFileMap, nil +} From 9e74753d5b391b0a013c9a41ea3b4d29019d91d5 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 2 Dec 2021 15:41:00 +0800 Subject: [PATCH 17/28] skip confirm --- pkg/cluster/manager/builder.go | 8 ++++---- pkg/cluster/manager/tls.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 2119684beb..fbcae97382 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -259,7 +259,6 @@ func buildScaleOutTask( if err != nil { return nil, err } - certificateTasks = append(certificateTasks, moniterCertificateTasks...) // monitor config monitorConfigTasks := buildInitMonitoredConfigTasks( @@ -288,7 +287,8 @@ func buildScaleOutTask( ParallelStep("+ Initialize target host environments", gOpt.Force, envInitTasks...). ParallelStep("+ Deploy TiDB instance", gOpt.Force, deployCompTasks...). ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). - ParallelStep("+ Generate scale-out config", gOpt.Force, scaleOutConfigTasks...) + ParallelStep("+ Generate scale-out config", gOpt.Force, scaleOutConfigTasks...). + ParallelStep("+ Init monitor config", gOpt.Force, moniterCertificateTasks...) } if afterDeploy != nil { @@ -361,7 +361,7 @@ func buildScaleConfigTasks( Data: dataDirs, Log: logDir, }, - ).BuildAsStep(fmt.Sprintf(" - Generate Scale Config %s -> %s", inst.ComponentName(), inst.ID())) + ).BuildAsStep(fmt.Sprintf(" - Generate scale-out config %s -> %s", inst.ComponentName(), inst.ID())) scaleConfigTasks = append(scaleConfigTasks, t) }) @@ -440,7 +440,7 @@ func buildMonitoredDeployTask( host, deployDir, ) - deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", comp, host))) + deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Deploy %s -> %s", comp, host))) } } return diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 864007114a..aa75552225 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -163,7 +163,7 @@ func getTLSFileMap(m *Manager, clusterName string, topo spec.Topology, if !enableTLS && cleanCertificate { // get: host: set(tlsdir) - delFileMap = getCleanupFiles(topo, false, false, cleanCertificate, []string{}, []string{}) + delFileMap = getCleanupFiles(topo, false, false, cleanCertificate, false, []string{}, []string{}) // build file list string delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(clusterName, spec.TLSCertKeyDir)) for host, fileList := range delFileMap { From 251111ec6dd7a3910727dd3bae092e378befa932 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 2 Dec 2021 15:47:23 +0800 Subject: [PATCH 18/28] perfect tips --- pkg/cluster/manager/builder.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index fbcae97382..2d8a303169 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -273,7 +273,6 @@ func buildScaleOutTask( gOpt, p, ) - scaleOutConfigTasks = append(scaleOutConfigTasks, monitorConfigTasks...) builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) if err != nil { @@ -288,7 +287,7 @@ func buildScaleOutTask( ParallelStep("+ Deploy TiDB instance", gOpt.Force, deployCompTasks...). ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). ParallelStep("+ Generate scale-out config", gOpt.Force, scaleOutConfigTasks...). - ParallelStep("+ Init monitor config", gOpt.Force, moniterCertificateTasks...) + ParallelStep("+ Init monitor config", gOpt.Force, monitorConfigTasks...) } if afterDeploy != nil { From 023eee604c77999c2655317dc3f36f4bafd84540 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 2 Dec 2021 15:48:07 +0800 Subject: [PATCH 19/28] fix --- pkg/cluster/manager/builder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 2d8a303169..332d6ad805 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -259,6 +259,7 @@ func buildScaleOutTask( if err != nil { return nil, err } + certificateTasks = append(certificateTasks, moniterCertificateTasks...) // monitor config monitorConfigTasks := buildInitMonitoredConfigTasks( From 5760ac8e9f9c9cb4fc6b993208d09daeb7c57598 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 2 Dec 2021 19:26:49 +0800 Subject: [PATCH 20/28] finish 1 pd cluster enable/disable tls --- components/cluster/command/tls.go | 2 +- pkg/cluster/manager/builder.go | 31 +++++++--- pkg/cluster/manager/cacert.go | 16 +++++ pkg/cluster/manager/tls.go | 40 ++++--------- pkg/cluster/operation/pd_member.go | 96 ++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 38 deletions(-) create mode 100644 pkg/cluster/operation/pd_member.go diff --git a/components/cluster/command/tls.go b/components/cluster/command/tls.go index ceb9d2ec7d..a0722a0b2b 100644 --- a/components/cluster/command/tls.go +++ b/components/cluster/command/tls.go @@ -1,4 +1,4 @@ -// Copyright 2020 PingCAP, Inc. +// Copyright 2021 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 332d6ad805..a9f84c12a8 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -19,11 +19,13 @@ import ( "path/filepath" "strings" + "github.com/fatih/color" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/environment" + "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -676,12 +678,22 @@ func buildTLSTask( name string, metadata spec.Metadata, gOpt operator.Options, + reloadCertificate bool, p *tui.SSHConnectionProps, delFileMap map[string]set.StringSet, ) (task.Task, error) { topo := metadata.GetTopology() base := metadata.GetBaseMeta() + // load certificate file + if topo.BaseTopo().GlobalOptions.TLSEnabled { + tlsDir := m.specManager.Path(name, spec.TLSCertKeyDir) + log.Infof("Reload certificate: %s", color.YellowString(tlsDir)) + if err := m.loadCertificate(name, topo.BaseTopo().GlobalOptions, reloadCertificate); err != nil { + return nil, err + } + } + certificateTasks, err := buildCertificateTasks(m, name, topo, base, gOpt, p) if err != nil { return nil, err @@ -734,7 +746,10 @@ func buildTLSTask( ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, moniterCertificateTasks...). - ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...) + ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...). + Func("Save meta", func(_ context.Context) error { + return m.specManager.SaveMeta(name, metadata) + }) // cleanup tls files only in tls disable if !topo.BaseTopo().GlobalOptions.TLSEnabled { @@ -747,14 +762,14 @@ func buildTLSTask( if err != nil { return nil, err } - builder.Func("Restart Cluster", func(ctx context.Context) error { - return operator.Restart(ctx, topo, gOpt, tlsCfg) - }) - // finally save metadata - builder.Func("Save meta", func(_ context.Context) error { - return m.specManager.SaveMeta(name, metadata) - }) + builder. + Func("Restart Cluster", func(ctx context.Context) error { + return operator.Restart(ctx, topo, gOpt, tlsCfg) + }). + Func("Reload PD Members", func(ctx context.Context) error { + return operator.SetPDMember(ctx, name, topo.BaseTopo().GlobalOptions.TLSEnabled, tlsCfg, metadata) + }) return builder.Build(), nil } diff --git a/pkg/cluster/manager/cacert.go b/pkg/cluster/manager/cacert.go index 56177cbae1..48cdfe751a 100644 --- a/pkg/cluster/manager/cacert.go +++ b/pkg/cluster/manager/cacert.go @@ -142,3 +142,19 @@ func (m *Manager) checkCertificate(clusterName string) error { } return nil } + +// loadCertificate +// certificate file exists and reload is true +// will reload certificate file +func (m *Manager) loadCertificate(clusterName string, globalOptions *spec.GlobalOptions, reload bool) error { + err := m.checkCertificate(clusterName) + + // no need to reload and the file already exists + if !reload && err == nil { + return nil + } + + _, err = m.genAndSaveCertificate(clusterName, globalOptions) + + return err +} diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index aa75552225..88ebf7a841 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -1,4 +1,4 @@ -// Copyright 2020 PingCAP, Inc. +// Copyright 2021 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -55,13 +55,6 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica globalOptions := topo.BaseTopo().GlobalOptions globalOptions.TLSEnabled = enable - // load certificate file - if enable { - if err := loadCertificate(m, name, globalOptions, reloadCertificate); err != nil { - return err - } - } - if err := checkTLSEnv(topo, name, base.Version, skipConfirm); err != nil { return err } @@ -86,7 +79,7 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica // Build the tls tasks t, err := buildTLSTask( - m, name, metadata, gOpt, sshProxyProps, delFileMap) + m, name, metadata, gOpt, reloadCertificate, sshProxyProps, delFileMap) if err != nil { return err } @@ -119,30 +112,19 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica return nil } -// loadCertificate -// certificate file exists and reload is true -// will reload certificate file -func loadCertificate(m *Manager, clusterName string, globalOptions *spec.GlobalOptions, reload bool) error { - err := m.checkCertificate(clusterName) - - // no need to reload and the file already exists - if !reload && err == nil { - return nil - } - - _, err = m.genAndSaveCertificate(clusterName, globalOptions) - - return err -} - // checkTLSEnv check tiflash vserson and show confirm func checkTLSEnv(topo spec.Topology, clusterName, version string, skipConfirm bool) error { // check tiflash version if clusterSpec, ok := topo.(*spec.Specification); ok { - if clusterSpec.GlobalOptions.TLSEnabled && - semver.Compare(version, "v4.0.5") < 0 && - len(clusterSpec.TiFlashServers) > 0 { - return fmt.Errorf("TiFlash %s is not supported in TLS enabled cluster", version) + if clusterSpec.GlobalOptions.TLSEnabled { + if semver.Compare(version, "v4.0.5") < 0 && len(clusterSpec.TiFlashServers) > 0 { + return fmt.Errorf("TiFlash %s is not supported in TLS enabled cluster", version) + } + } + + if len(clusterSpec.PDServers) != 1 { + return errorx.EnsureStackTrace(fmt.Errorf("Multiple PD nodes is not supported enable/disable TLS")). + WithProperty(tui.SuggestionFromString("Please `scale-in` PD nodes to one and try again.")) } } diff --git a/pkg/cluster/operation/pd_member.go b/pkg/cluster/operation/pd_member.go new file mode 100644 index 0000000000..f0171c7aad --- /dev/null +++ b/pkg/cluster/operation/pd_member.go @@ -0,0 +1,96 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + "context" + "crypto/tls" + "fmt" + "strings" + "time" + + "github.com/pingcap/tiup/pkg/cluster/spec" +) + +// UpdatePDMember is used to update pd cluster member +type UpdatePDMember struct { + cluster string + tlsCfg *tls.Config + metadata spec.Metadata + enableTLS bool +} + +// SetPDMember set the member of pd-etcd +func SetPDMember(ctx context.Context, clusterName string, enable bool, tlsCfg *tls.Config, meta spec.Metadata) error { + u := &UpdatePDMember{ + cluster: clusterName, + tlsCfg: tlsCfg, + metadata: meta, + enableTLS: enable, + } + + return u.Execute(ctx) +} + +// Execute implements the Task interface +func (u *UpdatePDMember) Execute(ctx context.Context) error { + // connection etcd + etcdClient, err := u.metadata.GetTopology().(*spec.Specification).GetEtcdClient(u.tlsCfg) + if err != nil { + return err + } + // etcd client defaults to wait forever + // if all pd were down, don't hang forever + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + // txn := etcdClient.Txn(ctx) + + memberList, err := etcdClient.MemberList(ctx) + if err != nil { + return err + } + + for _, member := range memberList.Members { + _, err := etcdClient.MemberUpdate(ctx, member.GetID(), u.updatePeerURLs(member.PeerURLs)) + if err != nil { + return err + } + } + + // get member list after update + memberList, err = etcdClient.MemberList(ctx) + if err != nil { + return err + } + for _, member := range memberList.Members { + fmt.Printf("\tUpdate %s peerURLs: %v\n", member.Name, member.PeerURLs) + } + + return nil +} + +// updatePeerURLs http->https or https->http +func (u *UpdatePDMember) updatePeerURLs(peerURLs []string) []string { + newPeerURLs := []string{} + + for _, url := range peerURLs { + if u.enableTLS { + newPeerURLs = append(newPeerURLs, strings.Replace(url, "http://", "https://", 1)) + } else { + newPeerURLs = append(newPeerURLs, strings.Replace(url, "https://", "http://", 1)) + } + } + + return newPeerURLs +} From d36ca86eb22dd8995739f82001b4be5c4b8d26e8 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 2 Dec 2021 22:10:47 +0800 Subject: [PATCH 21/28] prevent duplicate set tls --- pkg/cluster/manager/tls.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 88ebf7a841..bc36ddc6cf 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -53,6 +53,14 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica // set tls_enabled globalOptions := topo.BaseTopo().GlobalOptions + if globalOptions.TLSEnabled == enable { + if enable { + log.Infof("cluster `%s` TLS status is already enable", name) + } else { + log.Infof("cluster `%s` TLS status is already disable", name) + } + return nil + } globalOptions.TLSEnabled = enable if err := checkTLSEnv(topo, name, base.Version, skipConfirm); err != nil { From 93e5ffab6a5558f20e690e2769f6e5b76c05fd56 Mon Sep 17 00:00:00 2001 From: srstack Date: Fri, 3 Dec 2021 11:41:10 +0800 Subject: [PATCH 22/28] perfect tips --- pkg/cluster/manager/builder.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index a9f84c12a8..badfd2fad9 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -688,7 +688,7 @@ func buildTLSTask( // load certificate file if topo.BaseTopo().GlobalOptions.TLSEnabled { tlsDir := m.specManager.Path(name, spec.TLSCertKeyDir) - log.Infof("Reload certificate: %s", color.YellowString(tlsDir)) + log.Infof("Generate certificate: %s", color.YellowString(tlsDir)) if err := m.loadCertificate(name, topo.BaseTopo().GlobalOptions, reloadCertificate); err != nil { return nil, err } @@ -744,8 +744,8 @@ func buildTLSTask( builder. ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). - ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, moniterCertificateTasks...). + ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...). Func("Save meta", func(_ context.Context) error { return m.specManager.SaveMeta(name, metadata) @@ -753,7 +753,7 @@ func buildTLSTask( // cleanup tls files only in tls disable if !topo.BaseTopo().GlobalOptions.TLSEnabled { - builder.Func("CleanupCluster", func(ctx context.Context) error { + builder.Func("Cleanup TLS files", func(ctx context.Context) error { return operator.CleanupComponent(ctx, delFileMap) }) } From d2b9b6a5438afc272b0ac76a797141ab403dc8b0 Mon Sep 17 00:00:00 2001 From: srstack Date: Fri, 3 Dec 2021 16:41:07 +0800 Subject: [PATCH 23/28] perfect tips --- pkg/cluster/manager/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index badfd2fad9..d95b0244bc 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -305,11 +305,11 @@ func buildScaleOutTask( // don't start the new instance if opt.Stage1 { // save scale out file lock - builder.Func("Create Scale-Out File Lock", func(_ context.Context) error { + builder.Func("Create scale-out file lock", func(_ context.Context) error { return m.specManager.NewScaleOutLock(name, newPart) }) } else { - builder.Func("Start Cluster", func(ctx context.Context) error { + builder.Func("Start new instances", func(ctx context.Context) error { return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg) }). ParallelStep("+ Refresh configs", false, refreshConfigTasks...). From 5b88b7f9c2b4bd57d56f851621fbacbf844b2c4b Mon Sep 17 00:00:00 2001 From: srstack Date: Sun, 5 Dec 2021 21:16:28 +0800 Subject: [PATCH 24/28] add force flag --- components/cluster/command/tls.go | 5 +++-- pkg/cluster/manager/tls.go | 3 ++- pkg/cluster/operation/operation.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/components/cluster/command/tls.go b/components/cluster/command/tls.go index a0722a0b2b..c23067c740 100644 --- a/components/cluster/command/tls.go +++ b/components/cluster/command/tls.go @@ -63,8 +63,9 @@ func newTLSCmd() *cobra.Command { }, } - cmd.Flags().BoolVar(&cleanCertificate, "clean-certificate", false, "Clean up the certificate file if it already exists when disable encrypted communication") - cmd.Flags().BoolVar(&reloadCertificate, "reload-certificate", false, "Reload the certificate file if it already exists when enable encrypted communication") + cmd.Flags().BoolVar(&cleanCertificate, "clean-certificate", false, "Cleanup the certificate file if it already exists when tls disable") + cmd.Flags().BoolVar(&reloadCertificate, "reload-certificate", false, "Load the certificate file whether it exists or not when tls enable") + cmd.Flags().BoolVar(&gOpt.Force, "force", false, "Force enable/disable tls regardless of the current state") return cmd } diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index bc36ddc6cf..0b3b2ad91d 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -53,7 +53,8 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica // set tls_enabled globalOptions := topo.BaseTopo().GlobalOptions - if globalOptions.TLSEnabled == enable { + // if force is true, skip this check + if globalOptions.TLSEnabled == enable && !gOpt.Force { if enable { log.Infof("cluster `%s` TLS status is already enable", name) } else { diff --git a/pkg/cluster/operation/operation.go b/pkg/cluster/operation/operation.go index 16555455eb..1cd42f839c 100644 --- a/pkg/cluster/operation/operation.go +++ b/pkg/cluster/operation/operation.go @@ -31,7 +31,7 @@ const ( type Options struct { Roles []string Nodes []string - Force bool // Option for upgrade subcommand + Force bool // Option for upgrade/tls subcommand SSHTimeout uint64 // timeout in seconds when connecting an SSH server OptTimeout uint64 // timeout in seconds for operations that support it, not to confuse with SSH timeout APITimeout uint64 // timeout in seconds for API operations that support it, like transferring store leader From 9828b3e935029f795290d6a49551920ea39fa761 Mon Sep 17 00:00:00 2001 From: srstack Date: Sun, 5 Dec 2021 21:28:07 +0800 Subject: [PATCH 25/28] add tls path with clean op --- pkg/cluster/manager/cleanup.go | 13 +++++++++++++ pkg/cluster/spec/profile.go | 19 ++++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index 9334d5fd48..861e0e7488 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -147,6 +147,7 @@ type cleanupFiles struct { cleanupAuditLog bool // whether to clean up the tidb server audit log retainDataRoles []string // roles that don't clean up retainDataNodes []string // roles that don't clean up + ansibleImport bool // cluster is ansible deploy delFileMap map[string]set.StringSet } @@ -230,6 +231,13 @@ func (c *cleanupFiles) instanceCleanupFiles(topo spec.Topology) { deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir()) tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) tlsPath.Insert(tlsDir) + + // ansible deploy + if ins.IsImported() { + ansibleTLSDir := filepath.Join(deployDir, spec.TLSCertKeyDirWithAnsible) + tlsPath.Insert(ansibleTLSDir) + c.ansibleImport = true + } } if c.delFileMap[ins.GetHost()] == nil { @@ -287,6 +295,11 @@ func (c *cleanupFiles) monitorCleanupFiles(topo spec.Topology) { if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) tlsPath.Insert(tlsDir) + // ansible deploy + if c.ansibleImport { + ansibleTLSDir := filepath.Join(deployDir, spec.TLSCertKeyDirWithAnsible) + tlsPath.Insert(ansibleTLSDir) + } } if c.delFileMap[host] == nil { diff --git a/pkg/cluster/spec/profile.go b/pkg/cluster/spec/profile.go index 813744f32b..cf852cb2e7 100644 --- a/pkg/cluster/spec/profile.go +++ b/pkg/cluster/spec/profile.go @@ -27,15 +27,16 @@ import ( // sub directory names const ( - TiUPPackageCacheDir = "packages" - TiUPClusterDir = "clusters" - TiUPAuditDir = "audit" - TLSCertKeyDir = "tls" - TLSCACert = "ca.crt" - TLSCAKey = "ca.pem" - TLSClientCert = "client.crt" - TLSClientKey = "client.pem" - PFXClientCert = "client.pfx" + TiUPPackageCacheDir = "packages" + TiUPClusterDir = "clusters" + TiUPAuditDir = "audit" + TLSCertKeyDir = "tls" + TLSCertKeyDirWithAnsible = "ssl" + TLSCACert = "ca.crt" + TLSCAKey = "ca.pem" + TLSClientCert = "client.crt" + TLSClientKey = "client.pem" + PFXClientCert = "client.pfx" ) var profileDir string From 3ddac959d420f5be370a5ebb0059cbf3742e0c30 Mon Sep 17 00:00:00 2001 From: srstack Date: Tue, 7 Dec 2021 15:20:38 +0800 Subject: [PATCH 26/28] m --- CHANGELOG.md | 28 +++++ cmd/mirror.go | 1 - cmd/root.go | 3 +- components/cluster/command/deploy.go | 4 +- components/cluster/command/display.go | 56 +--------- components/cluster/command/import.go | 23 +++- components/cluster/command/root.go | 26 ++--- components/dm/ansible/import_test.go | 7 +- components/dm/command/deploy.go | 2 +- components/dm/command/import.go | 2 +- components/dm/command/root.go | 16 ++- components/dm/command/scale_in.go | 1 - components/dm/spec/logic.go | 9 +- components/dm/spec/topology_dm.go | 5 +- components/playground/instance/tiflash.go | 2 +- components/playground/main.go | 4 +- components/playground/playground.go | 16 ++- pkg/checkpoint/checkpoint.go | 3 +- pkg/cluster/ansible/config.go | 18 ++-- pkg/cluster/ansible/import.go | 11 +- pkg/cluster/ansible/import_test.go | 6 +- pkg/cluster/ansible/inventory.go | 65 +++++------ pkg/cluster/ansible/service.go | 8 +- pkg/cluster/api/pdapi.go | 83 ++++++++------ pkg/cluster/ctxt/context.go | 42 +++++--- pkg/cluster/executor/local_test.go | 5 +- pkg/cluster/manager/basic.go | 45 +++++--- pkg/cluster/manager/builder.go | 125 ++++++++++++++++++---- pkg/cluster/manager/check.go | 48 ++++++--- pkg/cluster/manager/cleanup.go | 21 ++-- pkg/cluster/manager/deploy.go | 55 +++++++--- pkg/cluster/manager/destroy.go | 24 +++-- pkg/cluster/manager/display.go | 109 +++++++++++++++---- pkg/cluster/manager/edit_config.go | 15 ++- pkg/cluster/manager/exec.go | 17 +-- pkg/cluster/manager/list.go | 6 +- pkg/cluster/manager/manager.go | 35 +++--- pkg/cluster/manager/patch.go | 9 +- pkg/cluster/manager/reload.go | 12 ++- pkg/cluster/manager/rename.go | 3 +- pkg/cluster/manager/scale_in.go | 16 +-- pkg/cluster/manager/scale_out.go | 27 +++-- pkg/cluster/manager/transfer.go | 16 +-- pkg/cluster/manager/upgrade.go | 16 +-- pkg/cluster/module/wait_for.go | 4 +- pkg/cluster/operation/action.go | 62 ++++++----- pkg/cluster/operation/check.go | 4 +- pkg/cluster/operation/destroy.go | 75 +++++++------ pkg/cluster/operation/scale_in.go | 21 ++-- pkg/cluster/operation/upgrade.go | 23 ++-- pkg/cluster/spec/alertmanager.go | 4 +- pkg/cluster/spec/cdc.go | 4 +- pkg/cluster/spec/drainer.go | 4 +- pkg/cluster/spec/grafana.go | 4 +- pkg/cluster/spec/grafana_test.go | 3 +- pkg/cluster/spec/instance.go | 20 ++-- pkg/cluster/spec/monitoring.go | 38 ++++++- pkg/cluster/spec/monitoring_test.go | 94 ++++++++++++++++ pkg/cluster/spec/pd.go | 20 ++-- pkg/cluster/spec/pump.go | 4 +- pkg/cluster/spec/server_config.go | 6 +- pkg/cluster/spec/spec.go | 13 ++- pkg/cluster/spec/tidb.go | 4 +- pkg/cluster/spec/tiflash.go | 8 +- pkg/cluster/spec/tikv.go | 30 +++--- pkg/cluster/spec/tispark.go | 8 +- pkg/cluster/task/builder.go | 28 ++--- pkg/cluster/task/init_config_test.go | 3 +- pkg/cluster/task/monitored_config.go | 5 +- pkg/cluster/task/shell.go | 5 +- pkg/cluster/task/step.go | 64 +++++------ pkg/cluster/task/task.go | 8 +- pkg/environment/env.go | 4 +- pkg/logger/audit.go | 4 +- pkg/logger/{log => printer}/log.go | 47 +++++--- pkg/logger/printer/logger.go | 96 +++++++++++++++++ pkg/logger/{log => printer}/verbose.go | 10 +- pkg/proxy/http_proxy.go | 9 +- pkg/proxy/proxy.go | 22 ++-- pkg/proxy/tcp_proxy.go | 17 ++- pkg/repository/mirror.go | 6 +- pkg/repository/model/model.go | 8 +- pkg/repository/v1_repository.go | 14 +-- pkg/tui/progress/display_props.go | 6 +- pkg/utils/user.go | 6 +- pkg/version/version.go | 2 +- server/handler/component.go | 12 +-- server/handler/handler.go | 8 +- server/handler/rotate.go | 6 +- server/handler/tarball.go | 6 +- server/main.go | 4 +- server/rotate/rotate_server.go | 4 +- server/router.go | 6 +- server/session/session.go | 6 +- server/static.go | 6 +- tests/tiup-dm/script/task/run.sh | 1 + tests/tiup/test_tiup.sh | 51 ++++----- 97 files changed, 1305 insertions(+), 667 deletions(-) rename pkg/logger/{log => printer}/log.go (78%) create mode 100644 pkg/logger/printer/logger.go rename pkg/logger/{log => printer}/verbose.go (80%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6376368ba..35406ac060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ TiUP Changelog +## [1.8.0] 2021-12-06 + +### New Features + +- Add `data-dir` support for TiCDC in `tiup-playground` ([#1631](https://github.com/pingcap/tiup/pull/1631), [@nexustar](https://github.com/nexustar)) +- Add support of using custom files as input of `edit-config`, and support dumping the current full config to a file with `show-config` command in `tiup-cluster` ([#1637](https://github.com/pingcap/tiup/pull/1637), [@haiboumich](https://github.com/haiboumich)) +- Add support of next-generation monitor (`ng-monitor`) in `tiup-playground` ([#1648](https://github.com/pingcap/tiup/pull/1648), [@nexustar](https://github.com/nexustar)) +- Add support of inserting custom `scrape_configs` to Prometheus configs in `tiup-cluster` ([#1641](https://github.com/pingcap/tiup/pull/1641), [@nexustar](https://github.com/nexustar)) +- [experimental] Support 2-staged scaling out for `tiup-cluster` ([#1638](https://github.com/pingcap/tiup/pull/1638) [#1642](https://github.com/pingcap/tiup/pull/1642), [@srstack](https://github.com/srstack)) + - Scaling out of a TiDB cluster can be devided with `--stage1` and `--stage2` arguments, the stage 1 deploys files and configs but not starting the new instances, and the stage 2 actually starts the new instances and reload necessary configs + - This could be useful if you want to modify config of the new instances or use a custom binary with `patch` **before** the first start of the new instances +- [experimental] Implement plain text output and support custom output writer for logs ([#1646](https://github.com/pingcap/tiup/pull/1646), [@AstroProfundis](https://github.com/AstroProfundis)) + +### Fixes + +- Fix incorrect progress bar displaying in some tasks ([#1624](https://github.com/pingcap/tiup/pull/1624), [@nexustar](https://github.com/nexustar)) +- Fix incorrect argument flags in `tiup-playground` ([#1635](https://github.com/pingcap/tiup/pull/1635), [@srstack](https://github.com/srstack)) +- Fix files of monitoring agents and TiDB audit log not cleaned with `clean` command of `tiup-cluster` ([#1643](https://github.com/pingcap/tiup/pull/1643) [#1644](https://github.com/pingcap/tiup/pull/1644), [@srstack](https://github.com/srstack)) +- Fix confirmation prompt in `scale-out` can not be skipped with `--yes` argument in `tiup-cluster` ([#1645](https://github.com/pingcap/tiup/pull/1645), [@srstack](https://github.com/srstack)) +- Fix directory conflict error in some circumstances even when node is marked as `ignore_exporter` ([#1649](https://github.com/pingcap/tiup/pull/1649), [@AstroProfundis](https://github.com/AstroProfundis)) +- Fix DM nodes not added to node_exporter target list in Prometheus config in `tiup-dm` ([#1654](https://github.com/pingcap/tiup/pull/1654), [@AstroProfundis](https://github.com/AstroProfundis)) + +### Improvements + +- Add significant warning when `--force` argument is set for `scale-in` command in `tiup-cluster` ([#1629](https://github.com/pingcap/tiup/pull/1629), [@AstroProfundis](https://github.com/AstroProfundis)) +- Add environment variables to skip topology sanity check in `scale-in` command in `tiup-cluster` ([#1627](https://github.com/pingcap/tiup/pull/1627), [@AstroProfundis](https://github.com/AstroProfundis)) +- Update examples to use `--without-monitor` instead of `--monitor` for `tiup-playground` ([#1639](https://github.com/pingcap/tiup/pull/1639), [@dveeden](https://github.com/dveeden)) + ## [1.7.0] 2021-11-15 ### New Features diff --git a/cmd/mirror.go b/cmd/mirror.go index e439a33087..9790e1475d 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -32,7 +32,6 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/repository/model" ru "github.com/pingcap/tiup/pkg/repository/utils" diff --git a/cmd/root.go b/cmd/root.go index b5b71bb4c9..38033a4b1a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/exec" "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/telemetry" @@ -41,6 +41,7 @@ var ( reportEnabled bool // is telemetry report enabled eventUUID = uuid.New().String() teleCommand string + log = logprinter.NewLogger("") // use default logger ) // arguments diff --git a/components/cluster/command/deploy.go b/components/cluster/command/deploy.go index 9402ef3f4a..15bf426f0b 100644 --- a/components/cluster/command/deploy.go +++ b/components/cluster/command/deploy.go @@ -80,7 +80,7 @@ func newDeploy() *cobra.Command { } func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Options) { - nodeInfoTask := task.NewBuilder(gOpt.DisplayMode).Func("Check status", func(ctx context.Context) error { + nodeInfoTask := task.NewBuilder(builder.Logger).Func("Check status", func(ctx context.Context) error { var err error teleNodeInfos, err = operator.GetNodeInfo(ctx, topo) _ = err @@ -92,7 +92,7 @@ func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Opt builder.ParallelStep("+ Check status", false, nodeInfoTask) } - enableTask := task.NewBuilder(gOpt.DisplayMode).Func("Setting service auto start on boot", func(ctx context.Context) error { + enableTask := task.NewBuilder(builder.Logger).Func("Setting service auto start on boot", func(ctx context.Context) error { return operator.Enable(ctx, topo, operator.Options{}, true) }).BuildAsStep("Enable service").SetHidden(true) diff --git a/components/cluster/command/display.go b/components/cluster/command/display.go index 48f57158a0..8e43adc3cc 100644 --- a/components/cluster/command/display.go +++ b/components/cluster/command/display.go @@ -14,17 +14,11 @@ package command import ( - "crypto/tls" "errors" "fmt" - "net/url" - "time" - "github.com/fatih/color" perrs "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/meta" "github.com/spf13/cobra" ) @@ -73,7 +67,7 @@ func newDisplayCmd() *cobra.Command { if err != nil { return err } - return displayDashboardInfo(clusterName, tlsCfg) + return cm.DisplayDashboardInfo(clusterName, tlsCfg) } if showTiKVLabels { return cm.DisplayTiKVLabels(clusterName, gOpt) @@ -91,51 +85,3 @@ func newDisplayCmd() *cobra.Command { return cmd } - -func displayDashboardInfo(clusterName string, tlsCfg *tls.Config) error { - metadata, err := spec.ClusterMetadata(clusterName) - if err != nil && !errors.Is(perrs.Cause(err), meta.ErrValidate) && - !errors.Is(perrs.Cause(err), spec.ErrNoTiSparkMaster) { - return err - } - - pdEndpoints := make([]string, 0) - for _, pd := range metadata.Topology.PDServers { - pdEndpoints = append(pdEndpoints, fmt.Sprintf("%s:%d", pd.Host, pd.ClientPort)) - } - - pdAPI := api.NewPDClient(pdEndpoints, 2*time.Second, tlsCfg) - dashboardAddr, err := pdAPI.GetDashboardAddress() - if err != nil { - return fmt.Errorf("failed to retrieve TiDB Dashboard instance from PD: %s", err) - } - if dashboardAddr == "auto" { - return fmt.Errorf("TiDB Dashboard is not initialized, please start PD and try again") - } else if dashboardAddr == "none" { - return fmt.Errorf("TiDB Dashboard is disabled") - } - - u, err := url.Parse(dashboardAddr) - if err != nil { - return fmt.Errorf("unknown TiDB Dashboard PD instance: %s", dashboardAddr) - } - - u.Path = "/dashboard/" - - if tlsCfg != nil { - fmt.Println( - "Client certificate:", - color.CyanString(tidbSpec.Path(clusterName, spec.TLSCertKeyDir, spec.PFXClientCert)), - ) - fmt.Println( - "Certificate password:", - color.CyanString(crypto.PKCS12Password), - ) - } - fmt.Println( - "Dashboard URL:", - color.CyanString(u.String()), - ) - - return nil -} diff --git a/components/cluster/command/import.go b/components/cluster/command/import.go index 3c7c8d250f..f153c20d6e 100644 --- a/components/cluster/command/import.go +++ b/components/cluster/command/import.go @@ -14,14 +14,15 @@ package command import ( + "context" "fmt" "os" "path/filepath" "github.com/fatih/color" "github.com/pingcap/tiup/pkg/cluster/ansible" + "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/tui" tiuputils "github.com/pingcap/tiup/pkg/utils" "github.com/spf13/cobra" @@ -49,8 +50,14 @@ func newImportCmd() *cobra.Command { ansibleDir = cwd } + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + log, + ) + // migrate cluster metadata from Ansible inventory - clsName, clsMeta, inv, err := ansible.ReadInventory(ansibleDir, inventoryFileName) + clsName, clsMeta, inv, err := ansible.ReadInventory(ctx, ansibleDir, inventoryFileName) if err != nil { return err } @@ -103,7 +110,15 @@ func newImportCmd() *cobra.Command { } // parse config and import nodes - if err = ansible.ParseAndImportInventory(ansibleDir, ansibleCfgFile, clsMeta, inv, gOpt.SSHTimeout, gOpt.SSHType); err != nil { + if err = ansible.ParseAndImportInventory( + ctx, + ansibleDir, + ansibleCfgFile, + clsMeta, + inv, + gOpt.SSHTimeout, + gOpt.SSHType, + ); err != nil { return err } @@ -123,7 +138,7 @@ func newImportCmd() *cobra.Command { } // copy config files form deployment servers - if err = ansible.ImportConfig(clsName, clsMeta, gOpt); err != nil { + if err = ansible.ImportConfig(ctx, clsName, clsMeta, gOpt); err != nil { return err } diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go index 33adeb7c94..82ad9319db 100644 --- a/components/cluster/command/root.go +++ b/components/cluster/command/root.go @@ -34,7 +34,7 @@ import ( tiupmeta "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/logger" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/telemetry" @@ -56,6 +56,7 @@ var ( teleNodeInfos []*telemetry.NodeInfo teleTopology string teleCommand []string + log = logprinter.NewLogger("") // init default logger ) var tidbSpec *spec.SpecManager @@ -100,14 +101,8 @@ func init() { SilenceErrors: true, Version: version.NewTiUPVersion().String(), PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - switch strings.ToLower(gOpt.DisplayMode) { - case "json": - log.SetDisplayMode(log.DisplayModeJSON) - case "plain", "text": - log.SetDisplayMode(log.DisplayModePlain) - default: - log.SetDisplayMode(log.DisplayModeDefault) - } + // populate logger + log.SetDisplayModeFromString(gOpt.DisplayMode) var err error var env *tiupmeta.Environment @@ -116,7 +111,7 @@ func init() { } tidbSpec = spec.GetSpecManager() - cm = manager.NewManager("tidb", tidbSpec, spec.TiDBComponentVersion) + cm = manager.NewManager("tidb", tidbSpec, spec.TiDBComponentVersion, log) logger.EnableAuditLog(spec.AuditDir()) // Running in other OS/ARCH Should be fine we only download manifest file. @@ -140,7 +135,14 @@ func init() { log.Infof("The --native-ssh flag has been deprecated, please use --ssh=system") } - err = proxy.MaybeStartProxy(gOpt.SSHProxyHost, gOpt.SSHProxyPort, gOpt.SSHProxyUser, gOpt.SSHProxyUsePassword, gOpt.SSHProxyIdentity) + err = proxy.MaybeStartProxy( + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + gOpt.SSHProxyUsePassword, + gOpt.SSHProxyIdentity, + log, + ) if err != nil { return perrs.Annotate(err, "start http-proxy") } @@ -350,7 +352,7 @@ func Execute() { } switch log.GetDisplayMode() { - case log.DisplayModeJSON: + case logprinter.DisplayModeJSON: obj := struct { Code int `json:"exit_code"` Err string `json:"error,omitempty"` diff --git a/components/dm/ansible/import_test.go b/components/dm/ansible/import_test.go index f3223ad26f..a3b4fe649f 100644 --- a/components/dm/ansible/import_test.go +++ b/components/dm/ansible/import_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tiup/components/dm/spec" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/executor" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/stretchr/testify/require" ) @@ -142,7 +143,11 @@ func TestImportFromAnsible(t *testing.T) { im, err := NewImporter(dir, "inventory.ini", executor.SSHTypeBuiltin, 0) assert.Nil(err) im.testExecutorGetter = &executorGetter{} - clusterName, meta, err := im.ImportFromAnsibleDir(ctxt.New(context.Background(), 0)) + clusterName, meta, err := im.ImportFromAnsibleDir(ctxt.New( + context.Background(), + 0, + logprinter.NewLogger(""), + )) assert.Nil(err, "verbose: %+v", err) assert.Equal("test-cluster", clusterName) diff --git a/components/dm/command/deploy.go b/components/dm/command/deploy.go index ee0654200c..4e7a165c38 100644 --- a/components/dm/command/deploy.go +++ b/components/dm/command/deploy.go @@ -82,7 +82,7 @@ func supportVersion(vs string) error { } func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Options) { - enableTask := task.NewBuilder(gOpt.DisplayMode).Func("Setting service auto start on boot", func(ctx context.Context) error { + enableTask := task.NewBuilder(builder.Logger).Func("Setting service auto start on boot", func(ctx context.Context) error { return operator.Enable(ctx, topo, operator.Options{}, true) }).BuildAsStep("Enable service").SetHidden(true) diff --git a/components/dm/command/import.go b/components/dm/command/import.go index f3ccc82ca5..70667d8ce9 100644 --- a/components/dm/command/import.go +++ b/components/dm/command/import.go @@ -49,7 +49,7 @@ func newImportCmd() *cobra.Command { return err } - ctx := ctxt.New(context.Background(), 0) + ctx := ctxt.New(context.Background(), 0, log) clusterName, meta, err := importer.ImportFromAnsibleDir(ctx) if err != nil { return err diff --git a/components/dm/command/root.go b/components/dm/command/root.go index 00e1cbabfe..cb8d007b7e 100644 --- a/components/dm/command/root.go +++ b/components/dm/command/root.go @@ -31,6 +31,7 @@ import ( tiupmeta "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/logger" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/tui" @@ -46,6 +47,7 @@ var ( rootCmd *cobra.Command gOpt operator.Options skipConfirm bool + log = logprinter.NewLogger("") // init default logger ) var dmspec *cspec.SpecManager @@ -72,6 +74,9 @@ please backup your data before process.`, SilenceErrors: true, Version: version.NewTiUPVersion().String(), PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + // populate logger + log.SetDisplayModeFromString(gOpt.DisplayMode) + var err error var env *tiupmeta.Environment if err = cspec.Initialize("dm"); err != nil { @@ -80,7 +85,7 @@ please backup your data before process.`, dmspec = spec.GetSpecManager() logger.EnableAuditLog(cspec.AuditDir()) - cm = manager.NewManager("dm", dmspec, spec.DMComponentVersion) + cm = manager.NewManager("dm", dmspec, spec.DMComponentVersion, log) // Running in other OS/ARCH Should be fine we only download manifest file. env, err = tiupmeta.InitEnv(repository.Options{ @@ -99,7 +104,14 @@ please backup your data before process.`, fmt.Println("The --native-ssh flag has been deprecated, please use --ssh=system") } - err = proxy.MaybeStartProxy(gOpt.SSHProxyHost, gOpt.SSHProxyPort, gOpt.SSHProxyUser, gOpt.SSHProxyUsePassword, gOpt.SSHProxyIdentity) + err = proxy.MaybeStartProxy( + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + gOpt.SSHProxyUsePassword, + gOpt.SSHProxyIdentity, + log, + ) if err != nil { return perrs.Annotate(err, "start http-proxy") } diff --git a/components/dm/command/scale_in.go b/components/dm/command/scale_in.go index f412ee925d..985f777aca 100644 --- a/components/dm/command/scale_in.go +++ b/components/dm/command/scale_in.go @@ -26,7 +26,6 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/set" "github.com/spf13/cobra" ) diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go index 1dca530670..11775a02ba 100644 --- a/components/dm/spec/logic.go +++ b/components/dm/spec/logic.go @@ -21,12 +21,10 @@ import ( "strings" "time" - "github.com/pingcap/tiup/pkg/logger/log" - "github.com/pingcap/tiup/pkg/meta" - "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/template/scripts" + "github.com/pingcap/tiup/pkg/meta" ) // Components names supported by TiUP @@ -91,7 +89,7 @@ func (c *DMMasterComponent) Instances() []Instance { s.DataDir, }, StatusFn: s.Status, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return spec.UptimeByHost(s.Host, s.Port, tlsCfg) }, }, @@ -182,7 +180,6 @@ func (i *MasterInstance) ScaleConfig( ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(c.Endpoints(deployUser)...) fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort())) - log.Infof("script path: %s", fp) if err := cfg.ConfigToFile(fp); err != nil { return err } @@ -233,7 +230,7 @@ func (c *DMWorkerComponent) Instances() []Instance { s.DataDir, }, StatusFn: s.Status, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return spec.UptimeByHost(s.Host, s.Port, tlsCfg) }, }, diff --git a/components/dm/spec/topology_dm.go b/components/dm/spec/topology_dm.go index ba2527e538..6870fe12fd 100644 --- a/components/dm/spec/topology_dm.go +++ b/components/dm/spec/topology_dm.go @@ -14,6 +14,7 @@ package spec import ( + "context" "crypto/tls" "fmt" "path/filepath" @@ -139,7 +140,7 @@ type MasterSpec struct { } // Status queries current status of the instance -func (s *MasterSpec) Status(tlsCfg *tls.Config, _ ...string) string { +func (s *MasterSpec) Status(_ context.Context, tlsCfg *tls.Config, _ ...string) string { addr := fmt.Sprintf("%s:%d", s.Host, s.Port) dc := api.NewDMMasterClient([]string{addr}, statusQueryTimeout, tlsCfg) isFound, isActive, isLeader, err := dc.GetMaster(s.Name) @@ -205,7 +206,7 @@ type WorkerSpec struct { } // Status queries current status of the instance -func (s *WorkerSpec) Status(tlsCfg *tls.Config, masterList ...string) string { +func (s *WorkerSpec) Status(_ context.Context, tlsCfg *tls.Config, masterList ...string) string { if len(masterList) < 1 { return "N/A" } diff --git a/components/playground/instance/tiflash.go b/components/playground/instance/tiflash.go index b63f9cf638..7691da4245 100644 --- a/components/playground/instance/tiflash.go +++ b/components/playground/instance/tiflash.go @@ -109,7 +109,7 @@ func (inst *TiFlashInstance) Start(ctx context.Context, version utils.Version) e } // Wait for PD - pdClient := api.NewPDClient(endpoints, 10*time.Second, nil) + pdClient := api.NewPDClient(ctx, endpoints, 10*time.Second, nil) // set low-space-ratio to 1 to avoid low disk space lowSpaceRatio, err := json.Marshal(scheduleConfig{ LowSpaceRatio: 0.99, diff --git a/components/playground/main.go b/components/playground/main.go index 4ac764fe64..e979883321 100644 --- a/components/playground/main.go +++ b/components/playground/main.go @@ -39,7 +39,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/telemetry" "github.com/pingcap/tiup/pkg/utils" @@ -74,6 +74,7 @@ var ( tag string tiupDataDir string dataDir string + log = logprinter.NewLogger("") ) const ( @@ -225,6 +226,7 @@ Examples: var booted uint32 ctx, cancel := context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, logprinter.ContextKeyLogger, log) defer cancel() go func() { diff --git a/components/playground/playground.go b/components/playground/playground.go index 25a86ab248..e334abe7f6 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/environment" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/tui/progress" "github.com/pingcap/tiup/pkg/utils" "golang.org/x/mod/semver" @@ -143,7 +144,10 @@ func (p *Playground) pdClient() *api.PDClient { addrs = append(addrs, inst.Addr()) } - return api.NewPDClient(addrs, 10*time.Second, nil) + return api.NewPDClient( + context.WithValue(context.TODO(), logprinter.ContextKeyLogger, log), + addrs, 10*time.Second, nil, + ) } func (p *Playground) killKVIfTombstone(inst *instance.TiKVInstance) { @@ -452,7 +456,10 @@ func (p *Playground) handleScaleOut(w io.Writer, cmd *Command) error { return err } - err = p.startInstance(context.TODO(), inst) + err = p.startInstance( + context.WithValue(context.TODO(), logprinter.ContextKeyLogger, log), + inst, + ) if err != nil { return err } @@ -710,7 +717,10 @@ func (p *Playground) waitAllTiFlashUp() { for _, pd := range p.pds { endpoints = append(endpoints, pd.Addr()) } - pdClient := api.NewPDClient(endpoints, 10*time.Second, nil) + pdClient := api.NewPDClient( + context.WithValue(context.TODO(), logprinter.ContextKeyLogger, log), + endpoints, 10*time.Second, nil, + ) var wg sync.WaitGroup bars := progress.NewMultiBar(color.YellowString("Waiting for tiflash instances ready\n")) diff --git a/pkg/checkpoint/checkpoint.go b/pkg/checkpoint/checkpoint.go index 56533ad7b2..91cbec2881 100644 --- a/pkg/checkpoint/checkpoint.go +++ b/pkg/checkpoint/checkpoint.go @@ -24,7 +24,6 @@ import ( "sync/atomic" "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/version" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -79,7 +78,7 @@ func Acquire(ctx context.Context, fs FieldSet, point map[string]interface{}) *Po if DebugCheckpoint { panic("the context passed to checkpoint.Acquire is not generated by checkpoint.NewContext") } - log.Debugf("context missing for checkpoint, the result of replaying this operation may be unexpected!") + zap.L().Debug("context missing for checkpoint, the result of replaying this operation may be unexpected!") ctx = NewContext(ctx) } diff --git a/pkg/cluster/ansible/config.go b/pkg/cluster/ansible/config.go index cfb3679eec..09bc5e5962 100644 --- a/pkg/cluster/ansible/config.go +++ b/pkg/cluster/ansible/config.go @@ -19,17 +19,16 @@ import ( "path/filepath" "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/tui" ) // ImportConfig copies config files from cluster which deployed through tidb-ansible -func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options) error { +func ImportConfig(ctx context.Context, name string, clsMeta *spec.ClusterMeta, gOpt operator.Options) error { // there may be already cluster dir, skip create // if err := os.MkdirAll(meta.ClusterPath(name), 0755); err != nil { // return err @@ -38,6 +37,7 @@ func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options) // return err // } + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) var sshProxyProps *tui.SSHConnectionProps = &tui.SSHConnectionProps{} if gOpt.SSHType != executor.SSHTypeNone && len(gOpt.SSHProxyHost) != 0 { var err error @@ -47,11 +47,11 @@ func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options) } var copyFileTasks []task.Task for _, comp := range clsMeta.Topology.ComponentsByStartOrder() { - log.Infof("Copying config file(s) of %s...", comp.Name()) + logger.Infof("Copying config file(s) of %s...", comp.Name()) for _, inst := range comp.Instances() { switch inst.ComponentName() { case spec.ComponentPD, spec.ComponentTiKV, spec.ComponentPump, spec.ComponentTiDB, spec.ComponentDrainer: - t := task.NewBuilder(""). + t := task.NewBuilder(logger). SSHKeySet( spec.ClusterPath(name, "ssh", "id_rsa"), spec.ClusterPath(name, "ssh", "id_rsa.pub")). @@ -85,7 +85,7 @@ func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options) Build() copyFileTasks = append(copyFileTasks, t) case spec.ComponentTiFlash: - t := task.NewBuilder(""). + t := task.NewBuilder(logger). SSHKeySet( spec.ClusterPath(name, "ssh", "id_rsa"), spec.ClusterPath(name, "ssh", "id_rsa.pub")). @@ -134,13 +134,13 @@ func ImportConfig(name string, clsMeta *spec.ClusterMeta, gOpt operator.Options) } } } - t := task.NewBuilder(""). + t := task.NewBuilder(logger). Parallel(false, copyFileTasks...). Build() - if err := t.Execute(ctxt.New(context.Background(), 0)); err != nil { + if err := t.Execute(ctx); err != nil { return errors.Trace(err) } - log.Infof("Finished copying configs.") + logger.Infof("Finished copying configs.") return nil } diff --git a/pkg/cluster/ansible/import.go b/pkg/cluster/ansible/import.go index b21c61037b..dd9ef7a6fd 100644 --- a/pkg/cluster/ansible/import.go +++ b/pkg/cluster/ansible/import.go @@ -14,6 +14,7 @@ package ansible import ( + "context" "fmt" "io" "io/fs" @@ -26,13 +27,15 @@ import ( "github.com/BurntSushi/toml" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "github.com/relex/aini" ) // ReadInventory reads the inventory files of a TiDB cluster deployed by TiDB-Ansible -func ReadInventory(dir, inventoryFileName string) (string, *spec.ClusterMeta, *aini.InventoryData, error) { +func ReadInventory(ctx context.Context, dir, inventoryFileName string) (string, *spec.ClusterMeta, *aini.InventoryData, error) { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + if inventoryFileName == "" { inventoryFileName = AnsibleInventoryFile } @@ -42,13 +45,13 @@ func ReadInventory(dir, inventoryFileName string) (string, *spec.ClusterMeta, *a } defer inventoryFile.Close() - log.Infof("Found inventory file %s, parsing...", inventoryFile.Name()) + logger.Infof("Found inventory file %s, parsing...", inventoryFile.Name()) clsName, clsMeta, inventory, err := parseInventoryFile(inventoryFile) if err != nil { return "", nil, inventory, err } - log.Infof("Found cluster \"%s\" (%s), deployed with user %s.", + logger.Infof("Found cluster \"%s\" (%s), deployed with user %s.", clsName, clsMeta.Version, clsMeta.User) return clsName, clsMeta, inventory, err } diff --git a/pkg/cluster/ansible/import_test.go b/pkg/cluster/ansible/import_test.go index 19d58a116b..f02a2b4bb6 100644 --- a/pkg/cluster/ansible/import_test.go +++ b/pkg/cluster/ansible/import_test.go @@ -14,6 +14,7 @@ package ansible import ( + "context" "fmt" "os" "path/filepath" @@ -24,6 +25,7 @@ import ( "github.com/creasty/defaults" . "github.com/pingcap/check" "github.com/pingcap/tiup/pkg/cluster/spec" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "gopkg.in/yaml.v3" ) @@ -147,7 +149,9 @@ func (s *ansSuite) TestParseGroupVars(c *C) { _, clsMeta, inv, err := parseInventoryFile(invData) c.Assert(err, IsNil) - err = parseGroupVars(dir, ansCfgFile, clsMeta, inv) + err = parseGroupVars(context.WithValue( + context.TODO(), logprinter.ContextKeyLogger, logprinter.NewLogger(""), + ), dir, ansCfgFile, clsMeta, inv) c.Assert(err, IsNil) err = defaults.Set(clsMeta) c.Assert(err, IsNil) diff --git a/pkg/cluster/ansible/inventory.go b/pkg/cluster/ansible/inventory.go index c457587ea4..bc88227000 100644 --- a/pkg/cluster/ansible/inventory.go +++ b/pkg/cluster/ansible/inventory.go @@ -14,6 +14,7 @@ package ansible import ( + "context" "os" "path/filepath" "strconv" @@ -22,7 +23,7 @@ import ( "github.com/creasty/defaults" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/relex/aini" "gopkg.in/ini.v1" "gopkg.in/yaml.v2" @@ -49,14 +50,14 @@ var ( ) // ParseAndImportInventory builds a basic ClusterMeta from the main Ansible inventory -func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini.InventoryData, sshTimeout uint64, sshType executor.SSHType) error { - if err := parseGroupVars(dir, ansCfgFile, clsMeta, inv); err != nil { +func ParseAndImportInventory(ctx context.Context, dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini.InventoryData, sshTimeout uint64, sshType executor.SSHType) error { + if err := parseGroupVars(ctx, dir, ansCfgFile, clsMeta, inv); err != nil { return err } for i := 0; i < len(clsMeta.Topology.TiDBServers); i++ { s := clsMeta.Topology.TiDBServers[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -64,7 +65,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.TiKVServers); i++ { s := clsMeta.Topology.TiKVServers[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -72,7 +73,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.PDServers); i++ { s := clsMeta.Topology.PDServers[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -80,7 +81,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.TiFlashServers); i++ { s := clsMeta.Topology.TiFlashServers[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -88,7 +89,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.PumpServers); i++ { s := clsMeta.Topology.PumpServers[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -96,7 +97,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.Drainers); i++ { s := clsMeta.Topology.Drainers[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -104,7 +105,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.Monitors); i++ { s := clsMeta.Topology.Monitors[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -112,7 +113,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.Alertmanagers); i++ { s := clsMeta.Topology.Alertmanagers[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -120,7 +121,7 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, } for i := 0; i < len(clsMeta.Topology.Grafanas); i++ { s := clsMeta.Topology.Grafanas[i] - ins, err := parseDirs(clsMeta.User, s, sshTimeout, sshType) + ins, err := parseDirs(ctx, clsMeta.User, s, sshTimeout, sshType) if err != nil { return err } @@ -131,7 +132,9 @@ func ParseAndImportInventory(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, return defaults.Set(clsMeta) } -func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini.InventoryData) error { +func parseGroupVars(ctx context.Context, dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini.InventoryData) error { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + // set global vars in group_vars/all.yml grpVarsAll, err := readGroupVars(dir, groupVarsGlobal) if err != nil { @@ -197,11 +200,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.LogDir = strings.Trim(logDir, "\"") } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.TiDBServers = append(clsMeta.Topology.TiDBServers, tmpIns) } - log.Infof("Imported %d TiDB node(s).", len(clsMeta.Topology.TiDBServers)) + logger.Infof("Imported %d TiDB node(s).", len(clsMeta.Topology.TiDBServers)) } // tikv_servers @@ -243,11 +246,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.LogDir = strings.Trim(logDir, "\"") } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.TiKVServers = append(clsMeta.Topology.TiKVServers, tmpIns) } - log.Infof("Imported %d TiKV node(s).", len(clsMeta.Topology.TiKVServers)) + logger.Infof("Imported %d TiKV node(s).", len(clsMeta.Topology.TiKVServers)) } // pd_servers @@ -292,11 +295,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.LogDir = strings.Trim(logDir, "\"") } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.PDServers = append(clsMeta.Topology.PDServers, tmpIns) } - log.Infof("Imported %d PD node(s).", len(clsMeta.Topology.PDServers)) + logger.Infof("Imported %d PD node(s).", len(clsMeta.Topology.PDServers)) } // tiflash_servers @@ -365,11 +368,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.TmpDir = tmpDir } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.TiFlashServers = append(clsMeta.Topology.TiFlashServers, tmpIns) } - log.Infof("Imported %d TiFlash node(s).", len(clsMeta.Topology.TiFlashServers)) + logger.Infof("Imported %d TiFlash node(s).", len(clsMeta.Topology.TiFlashServers)) } // spark_master @@ -413,11 +416,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.Retention = _retention } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.Monitors = append(clsMeta.Topology.Monitors, tmpIns) } - log.Infof("Imported %d monitoring node(s).", len(clsMeta.Topology.Monitors)) + logger.Infof("Imported %d monitoring node(s).", len(clsMeta.Topology.Monitors)) } // monitored_servers @@ -456,11 +459,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.ClusterPort, _ = strconv.Atoi(clusterPort) } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.Alertmanagers = append(clsMeta.Topology.Alertmanagers, tmpIns) } - log.Infof("Imported %d Alertmanager node(s).", len(clsMeta.Topology.Alertmanagers)) + logger.Infof("Imported %d Alertmanager node(s).", len(clsMeta.Topology.Alertmanagers)) } // grafana_servers @@ -497,11 +500,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.Password = strings.Trim(passwd, "\"") } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.Grafanas = append(clsMeta.Topology.Grafanas, tmpIns) } - log.Infof("Imported %d Grafana node(s).", len(clsMeta.Topology.Grafanas)) + logger.Infof("Imported %d Grafana node(s).", len(clsMeta.Topology.Grafanas)) } // kafka_exporter_servers @@ -541,11 +544,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.LogDir = strings.Trim(logDir, "\"") } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.PumpServers = append(clsMeta.Topology.PumpServers, tmpIns) } - log.Infof("Imported %d Pump node(s).", len(clsMeta.Topology.PumpServers)) + logger.Infof("Imported %d Pump node(s).", len(clsMeta.Topology.PumpServers)) } // drainer_servers @@ -577,11 +580,11 @@ func parseGroupVars(dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini tmpIns.Port, _ = strconv.Atoi(port) } - log.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) + logger.Debugf("Imported %s node %s:%d.", tmpIns.Role(), tmpIns.Host, tmpIns.GetMainPort()) clsMeta.Topology.Drainers = append(clsMeta.Topology.Drainers, tmpIns) } - log.Infof("Imported %d Drainer node(s).", len(clsMeta.Topology.Drainers)) + logger.Infof("Imported %d Drainer node(s).", len(clsMeta.Topology.Drainers)) } // TODO: node_exporter and blackbox_exporter on custom port is not supported yet diff --git a/pkg/cluster/ansible/service.go b/pkg/cluster/ansible/service.go index 78aab1f75e..7b98161516 100644 --- a/pkg/cluster/ansible/service.go +++ b/pkg/cluster/ansible/service.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" ) var ( @@ -34,9 +34,9 @@ var ( ) // parseDirs sets values of directories of component -func parseDirs(user string, ins spec.InstanceSpec, sshTimeout uint64, sshType executor.SSHType) (spec.InstanceSpec, error) { +func parseDirs(ctx context.Context, user string, ins spec.InstanceSpec, sshTimeout uint64, sshType executor.SSHType) (spec.InstanceSpec, error) { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) hostName, sshPort := ins.SSH() - ctx := ctxt.New(context.Background(), 0) e, err := executor.New(sshType, false, executor.SSHConfig{ Host: hostName, @@ -48,7 +48,7 @@ func parseDirs(user string, ins spec.InstanceSpec, sshTimeout uint64, sshType ex if err != nil { return nil, err } - log.Debugf("Detecting deploy paths on %s...", hostName) + logger.Debugf("Detecting deploy paths on %s...", hostName) stdout, err := readStartScript(ctx, e, ins.Role(), hostName, ins.GetMainPort()) if len(stdout) <= 1 || err != nil { diff --git a/pkg/cluster/api/pdapi.go b/pkg/cluster/api/pdapi.go index abdcfb6d96..50946f5033 100644 --- a/pkg/cluster/api/pdapi.go +++ b/pkg/cluster/api/pdapi.go @@ -30,7 +30,7 @@ import ( perrs "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/utils" "golang.org/x/mod/semver" ) @@ -41,6 +41,7 @@ type PDClient struct { addrs []string tlsEnabled bool httpClient *utils.HTTPClient + ctx context.Context } // LabelInfo represents an instance label info @@ -56,28 +57,43 @@ type LabelInfo struct { Labels string `json:"labels"` } -// NewPDClient returns a new PDClient -func NewPDClient(addrs []string, timeout time.Duration, tlsConfig *tls.Config) *PDClient { +// NewPDClient returns a new PDClient, the context must have +// a *logprinter.Logger as value of "logger" +func NewPDClient( + ctx context.Context, + addrs []string, + timeout time.Duration, + tlsConfig *tls.Config, +) *PDClient { enableTLS := false if tlsConfig != nil { enableTLS = true } + if _, ok := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger); !ok { + panic("the context must have logger inside") + } + cli := &PDClient{ addrs: addrs, tlsEnabled: enableTLS, httpClient: utils.NewHTTPClient(timeout, tlsConfig), + ctx: ctx, } cli.tryIdentifyVersion() return cli } +func (pc *PDClient) l() *logprinter.Logger { + return pc.ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) +} + func (pc *PDClient) tryIdentifyVersion() { endpoints := pc.getEndpoints(pdVersionURI) response := map[string]string{} _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -165,7 +181,7 @@ func (pc *PDClient) CheckHealth() error { endpoints := pc.getEndpoints(pdPingURI) _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -189,7 +205,7 @@ func (pc *PDClient) GetStores() (*StoresInfo, error) { storesInfo := StoresInfo{} _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -267,7 +283,7 @@ func (pc *PDClient) WaitLeader(retryOpt *utils.RetryOption) error { } // return error by default, to make the retry work - log.Debugf("Still waitting for the PD leader to be elected") + pc.l().Debugf("Still waitting for the PD leader to be elected") return perrs.New("still waitting for the PD leader to be elected") }, *retryOpt); err != nil { return fmt.Errorf("error getting PD leader, %v", err) @@ -282,7 +298,7 @@ func (pc *PDClient) GetLeader() (*pdpb.Member, error) { leader := pdpb.Member{} _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -303,7 +319,7 @@ func (pc *PDClient) GetMembers() (*pdpb.GetMembersResponse, error) { members := pdpb.GetMembersResponse{} _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -327,7 +343,7 @@ func (pc *PDClient) GetConfig() (map[string]interface{}, error) { pdConfig := map[string]interface{}{} _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -347,7 +363,7 @@ func (pc *PDClient) GetClusterID() (int64, error) { clusterID := map[string]interface{}{} _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -384,7 +400,7 @@ func (pc *PDClient) EvictPDLeader(retryOpt *utils.RetryOption) error { } if len(members.Members) == 1 { - log.Warnf("Only 1 member in the PD cluster, skip leader evicting") + pc.l().Warnf("Only 1 member in the PD cluster, skip leader evicting") return nil } @@ -393,7 +409,7 @@ func (pc *PDClient) EvictPDLeader(retryOpt *utils.RetryOption) error { endpoints := pc.getEndpoints(cmd) _, err = tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Post(context.TODO(), endpoint, nil) + body, err := pc.httpClient.Post(pc.ctx, endpoint, nil) if err != nil { return body, err } @@ -423,7 +439,7 @@ func (pc *PDClient) EvictPDLeader(retryOpt *utils.RetryOption) error { } // return error by default, to make the retry work - log.Debugf("Still waitting for the PD leader to transfer") + pc.l().Debugf("Still waitting for the PD leader to transfer") return perrs.New("still waitting for the PD leader to transfer") }, *retryOpt); err != nil { return fmt.Errorf("error evicting PD leader, %v", err) @@ -459,7 +475,7 @@ func (pc *PDClient) EvictStoreLeader(host string, retryOpt *utils.RetryOption, c return nil } - log.Infof("\tEvicting %d leaders from store %s...", leaderCount, latestStore.Store.Address) + pc.l().Infof("\tEvicting %d leaders from store %s...", leaderCount, latestStore.Store.Address) // set scheduler for stores scheduler, err := json.Marshal(pdSchedulerRequest{ @@ -473,7 +489,7 @@ func (pc *PDClient) EvictStoreLeader(host string, retryOpt *utils.RetryOption, c endpoints := pc.getEndpoints(pdSchedulersURI) _, err = tryURLs(endpoints, func(endpoint string) ([]byte, error) { - return pc.httpClient.Post(context.TODO(), endpoint, bytes.NewBuffer(scheduler)) + return pc.httpClient.Post(pc.ctx, endpoint, bytes.NewBuffer(scheduler)) }) if err != nil { return err @@ -502,7 +518,7 @@ func (pc *PDClient) EvictStoreLeader(host string, retryOpt *utils.RetryOption, c if leaderCount == 0 { return nil } - log.Infof( + pc.l().Infof( "\t Still waitting for %d store leaders to transfer...", leaderCount, ) @@ -532,23 +548,24 @@ func (pc *PDClient) RemoveStoreEvict(host string) error { ) endpoints := pc.getEndpoints(cmd) + logger := pc.l() _, err = tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, statusCode, err := pc.httpClient.Delete(context.TODO(), endpoint, nil) + body, statusCode, err := pc.httpClient.Delete(pc.ctx, endpoint, nil) if err != nil { if statusCode == http.StatusNotFound || bytes.Contains(body, []byte("scheduler not found")) { - log.Debugf("Store leader evicting scheduler does not exist, ignore.") + logger.Debugf("Store leader evicting scheduler does not exist, ignore.") return body, nil } return body, err } - log.Debugf("Delete leader evicting scheduler of store %d success", latestStore.Store.Id) + logger.Debugf("Delete leader evicting scheduler of store %d success", latestStore.Store.Id) return body, nil }) if err != nil { return err } - log.Debugf("Removed store leader evicting scheduler from %s.", latestStore.Store.Address) + logger.Debugf("Removed store leader evicting scheduler from %s.", latestStore.Store.Address) return nil } @@ -567,16 +584,17 @@ func (pc *PDClient) DelPD(name string, retryOpt *utils.RetryOption) error { cmd := fmt.Sprintf("%s/name/%s", pdMembersURI, name) endpoints := pc.getEndpoints(cmd) + logger := pc.l() _, err = tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, statusCode, err := pc.httpClient.Delete(context.TODO(), endpoint, nil) + body, statusCode, err := pc.httpClient.Delete(pc.ctx, endpoint, nil) if err != nil { if statusCode == http.StatusNotFound || bytes.Contains(body, []byte("not found, pd")) { - log.Debugf("PD node does not exist, ignore: %s", body) + logger.Debugf("PD node does not exist, ignore: %s", body) return body, nil } return body, err } - log.Debugf("Delete PD %s from the cluster success", name) + logger.Debugf("Delete PD %s from the cluster success", name) return body, nil }) if err != nil { @@ -654,16 +672,17 @@ func (pc *PDClient) DelStore(host string, retryOpt *utils.RetryOption) error { cmd := fmt.Sprintf("%s/%d", pdStoreURI, storeID) endpoints := pc.getEndpoints(cmd) + logger := pc.l() _, err = tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, statusCode, err := pc.httpClient.Delete(context.TODO(), endpoint, nil) + body, statusCode, err := pc.httpClient.Delete(pc.ctx, endpoint, nil) if err != nil { if statusCode == http.StatusNotFound || bytes.Contains(body, []byte("not found")) { - log.Debugf("store %d %s does not exist, ignore: %s", storeID, host, body) + logger.Debugf("store %d %s does not exist, ignore: %s", storeID, host, body) return body, nil } return body, err } - log.Debugf("Delete store %d %s from the cluster success", storeID, host) + logger.Debugf("Delete store %d %s from the cluster success", storeID, host) return body, nil }) if err != nil { @@ -708,7 +727,7 @@ func (pc *PDClient) DelStore(host string, retryOpt *utils.RetryOption) error { func (pc *PDClient) updateConfig(url string, body io.Reader) error { endpoints := pc.getEndpoints(url) _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - return pc.httpClient.Post(context.TODO(), endpoint, body) + return pc.httpClient.Post(pc.ctx, endpoint, body) }) return err } @@ -722,7 +741,7 @@ func (pc *PDClient) UpdateReplicateConfig(body io.Reader) error { func (pc *PDClient) GetReplicateConfig() ([]byte, error) { endpoints := pc.getEndpoints(pdConfigReplicate) return tryURLs(endpoints, func(endpoint string) ([]byte, error) { - return pc.httpClient.Get(context.TODO(), endpoint) + return pc.httpClient.Get(pc.ctx, endpoint) }) } @@ -800,7 +819,7 @@ func (pc *PDClient) CheckRegion(state string) (*RegionsInfo, error) { regionsInfo := RegionsInfo{} _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { - body, err := pc.httpClient.Get(context.TODO(), endpoint) + body, err := pc.httpClient.Get(pc.ctx, endpoint) if err != nil { return body, err } @@ -823,7 +842,7 @@ func (pc *PDClient) SetReplicationConfig(key string, value int) error { if err != nil { return err } - log.Debugf("setting replication config: %s=%d", key, value) + pc.l().Debugf("setting replication config: %s=%d", key, value) return pc.updateConfig(pdReplicationModeURI, bytes.NewBuffer(body)) } @@ -840,6 +859,6 @@ func (pc *PDClient) SetAllStoreLimits(value int) error { if err != nil { return err } - log.Debugf("setting store limit: %d", value) + pc.l().Debugf("setting store limit: %d", value) return pc.updateConfig(pdStoresLimitURI, bytes.NewBuffer(body)) } diff --git a/pkg/cluster/ctxt/context.go b/pkg/cluster/ctxt/context.go index 219a877dd9..bca09eb71a 100644 --- a/pkg/cluster/ctxt/context.go +++ b/pkg/cluster/ctxt/context.go @@ -20,6 +20,7 @@ import ( "time" "github.com/pingcap/tiup/pkg/checkpoint" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/utils/mock" ) @@ -80,29 +81,36 @@ type ( ) // New create a context instance. -func New(ctx context.Context, limit int) context.Context { +func New(ctx context.Context, limit int, logger *logprinter.Logger) context.Context { concurrency := runtime.NumCPU() if limit > 0 { concurrency = limit } - ctx = checkpoint.NewContext(ctx) - return context.WithValue(ctx, ctxKey, &Context{ - mutex: sync.RWMutex{}, - Ev: NewEventBus(), - exec: struct { - executors map[string]Executor - stdouts map[string][]byte - stderrs map[string][]byte - checkResults map[string][]interface{} - }{ - executors: make(map[string]Executor), - stdouts: make(map[string][]byte), - stderrs: make(map[string][]byte), - checkResults: make(map[string][]interface{}), + return context.WithValue( + context.WithValue( + checkpoint.NewContext(ctx), + logprinter.ContextKeyLogger, + logger, + ), + ctxKey, + &Context{ + mutex: sync.RWMutex{}, + Ev: NewEventBus(), + exec: struct { + executors map[string]Executor + stdouts map[string][]byte + stderrs map[string][]byte + checkResults map[string][]interface{} + }{ + executors: make(map[string]Executor), + stdouts: make(map[string][]byte), + stderrs: make(map[string][]byte), + checkResults: make(map[string][]interface{}), + }, + Concurrency: concurrency, // default to CPU count }, - Concurrency: concurrency, // default to CPU count - }) + ) } // GetInner return *Context from context.Context's value diff --git a/pkg/cluster/executor/local_test.go b/pkg/cluster/executor/local_test.go index 870b5e6e12..60fe8141b0 100644 --- a/pkg/cluster/executor/local_test.go +++ b/pkg/cluster/executor/local_test.go @@ -21,11 +21,12 @@ import ( "testing" "github.com/pingcap/tiup/pkg/cluster/ctxt" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/stretchr/testify/require" ) func TestLocal(t *testing.T) { - ctx := ctxt.New(context.Background(), 0) + ctx := ctxt.New(context.Background(), 0, logprinter.NewLogger("")) assert := require.New(t) user, err := user.Current() @@ -72,7 +73,7 @@ func TestWrongIP(t *testing.T) { } func TestLocalExecuteWithQuotes(t *testing.T) { - ctx := ctxt.New(context.Background(), 0) + ctx := ctxt.New(context.Background(), 0, logprinter.NewLogger("")) assert := require.New(t) user, err := user.Current() diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index 26caf29b2d..0d996e457b 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -26,7 +26,6 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -35,9 +34,9 @@ import ( // EnableCluster enable/disable the service in a cluster func (m *Manager) EnableCluster(name string, gOpt operator.Options, isEnable bool) error { if isEnable { - log.Infof("Enabling cluster %s...", name) + m.logger.Infof("Enabling cluster %s...", name) } else { - log.Infof("Disabling cluster %s...", name) + m.logger.Infof("Disabling cluster %s...", name) } metadata, err := m.meta(name) @@ -65,7 +64,12 @@ func (m *Manager) EnableCluster(name string, gOpt operator.Options, isEnable boo t := b.Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -74,9 +78,9 @@ func (m *Manager) EnableCluster(name string, gOpt operator.Options, isEnable boo } if isEnable { - log.Infof("Enabled cluster `%s` successfully", name) + m.logger.Infof("Enabled cluster `%s` successfully", name) } else { - log.Infof("Disabled cluster `%s` successfully", name) + m.logger.Infof("Disabled cluster `%s` successfully", name) } return nil @@ -84,7 +88,7 @@ func (m *Manager) EnableCluster(name string, gOpt operator.Options, isEnable boo // StartCluster start the cluster with specified name. func (m *Manager) StartCluster(name string, gOpt operator.Options, fn ...func(b *task.Builder, metadata spec.Metadata)) error { - log.Infof("Starting cluster %s...", name) + m.logger.Infof("Starting cluster %s...", name) // check locked if err := m.specManager.ScaleOutLockedErr(name); err != nil { @@ -119,7 +123,12 @@ func (m *Manager) StartCluster(name string, gOpt operator.Options, fn ...func(b t := b.Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -127,7 +136,7 @@ func (m *Manager) StartCluster(name string, gOpt operator.Options, fn ...func(b return perrs.Trace(err) } - log.Infof("Started cluster `%s` successfully", name) + m.logger.Infof("Started cluster `%s` successfully", name) return nil } @@ -174,7 +183,12 @@ func (m *Manager) StopCluster(name string, gOpt operator.Options, skipConfirm bo }). Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -182,7 +196,7 @@ func (m *Manager) StopCluster(name string, gOpt operator.Options, skipConfirm bo return perrs.Trace(err) } - log.Infof("Stopped cluster `%s` successfully", name) + m.logger.Infof("Stopped cluster `%s` successfully", name) return nil } @@ -228,7 +242,12 @@ func (m *Manager) RestartCluster(name string, gOpt operator.Options, skipConfirm }). Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -236,7 +255,7 @@ func (m *Manager) RestartCluster(name string, gOpt operator.Options, skipConfirm return perrs.Trace(err) } - log.Infof("Restarted cluster `%s` successfully", name) + m.logger.Infof("Restarted cluster `%s` successfully", name) return nil } diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index d95b0244bc..7359e8e49d 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/environment" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -33,7 +33,12 @@ import ( ) // buildReloadPromTasks reloads Prometheus configuration -func buildReloadPromTasks(topo spec.Topology, gOpt operator.Options, nodes ...string) []task.Task { +func buildReloadPromTasks( + topo spec.Topology, + logger *logprinter.Logger, + gOpt operator.Options, + nodes ...string, +) []task.Task { monitor := spec.FindComponent(topo, spec.ComponentPrometheus) if monitor == nil { return nil @@ -48,7 +53,7 @@ func buildReloadPromTasks(topo spec.Topology, gOpt operator.Options, nodes ...st if deletedNodes.Exist(inst.ID()) { continue } - t := task.NewBuilder(gOpt.DisplayMode). + t := task.NewBuilder(logger). SystemCtl(inst.GetHost(), inst.ServiceName(), "reload", true). Build() tasks = append(tasks, t) @@ -117,7 +122,7 @@ func buildScaleOutTask( } } - t := task.NewBuilder(gOpt.DisplayMode). + t := task.NewBuilder(m.logger). RootSSH( instance.GetHost(), instance.GetSSHPort(), @@ -144,7 +149,15 @@ func buildScaleOutTask( }) // Download missing component - downloadCompTasks = buildDownloadCompTasks(base.Version, newPart, gOpt, m.bindVersion) + + downloadCompTasks = convertStepDisplaysToTasks(buildDownloadCompTasks( + base.Version, + newPart, + m.logger, + gOpt, + m.bindVersion, + )) + sshType := topo.BaseTopo().GlobalOptions.SSHType @@ -165,7 +178,25 @@ func buildScaleOutTask( filepath.Join(deployDir, "scripts"), } // Deploy component - tb := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, sshType). + + // tb := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, sshType). + tb := task.NewBuilder(m.logger). + UserSSH( + inst.GetHost(), + inst.GetSSHPort(), + base.User, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + p.Password, + p.IdentityFile, + p.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + sshType, + ). Mkdir(base.User, inst.GetHost(), deployDirs...). Mkdir(base.User, inst.GetHost(), dataDirs...). Mkdir(base.User, inst.GetHost(), logDir) @@ -207,22 +238,44 @@ func buildScaleOutTask( return nil, iterErr } + certificateTasks, err := buildCertificateTasks(m, name, newPart, base, gOpt, p) if err != nil { return nil, err } + hasImported := false + noAgentHosts := set.NewStringSet() + + mergedTopo.IterInstance(func(inst spec.Instance) { + deployDir := spec.Abs(base.User, inst.DeployDir()) + // data dir would be empty for components which don't need it + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(base.User, inst.LogDir()) + + // Download and copy the latest component to remote if the cluster is imported from Ansible + tb := task.NewBuilder(m.logger) + if inst.IsImported() { + switch compName := inst.ComponentName(); compName { + case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: + version := m.bindVersion(compName, base.Version) + tb.Download(compName, inst.OS(), inst.Arch(), version). + CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) + } + hasImported = true + } + // init scale out config scaleOutConfigTasks := buildScaleConfigTasks(m, name, topo, newPart, base, gOpt, p) // always ignore config check result in scale out gOpt.IgnoreConfigCheck = true refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, mergedTopo, base, gOpt, nil) - // handle dir scheme changes if hasImported { if err := spec.HandleImportPathMigration(name); err != nil { - return task.NewBuilder(gOpt.DisplayMode).Build(), err + return task.NewBuilder(m.logger).Build(), err } } @@ -312,8 +365,7 @@ func buildScaleOutTask( builder.Func("Start new instances", func(ctx context.Context) error { return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg) }). - ParallelStep("+ Refresh configs", false, refreshConfigTasks...). - Parallel(false, buildReloadPromTasks(metadata.GetTopology(), gOpt)...) + ParallelStep("+ Refresh configs", false, refreshConfigTasks...) } // remove scale-out file lock @@ -406,7 +458,7 @@ func buildMonitoredDeployTask( key := fmt.Sprintf("%s-%s-%s", comp, info.os, info.arch) if found := uniqueCompOSArch.Exist(key); !found { uniqueCompOSArch.Insert(key) - downloadCompTasks = append(downloadCompTasks, task.NewBuilder(gOpt.DisplayMode). + downloadCompTasks = append(downloadCompTasks, task.NewBuilder(m.logger). Download(comp, info.os, info.arch, version). BuildAsStep(fmt.Sprintf(" - Download %s:%s (%s/%s)", comp, version, info.os, info.arch))) } @@ -431,7 +483,25 @@ func buildMonitoredDeployTask( } // Deploy component - tb := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). + + // tb := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). + tb := task.NewBuilder(m.logger). + UserSSH( + host, + info.ssh, + globalOptions.User, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + p.Password, + p.IdentityFile, + p.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + globalOptions.SSHType, + ). Mkdir(globalOptions.User, host, deployDirs...). CopyComponent( comp, @@ -516,6 +586,7 @@ func buildInitMonitoredConfigTasks( noAgentHosts set.StringSet, globalOptions spec.GlobalOptions, monitoredOptions *spec.MonitoredOptions, + logger *logprinter.Logger, sshTimeout, exeTimeout uint64, gOpt operator.Options, p *tui.SSHConnectionProps, @@ -542,7 +613,24 @@ func buildInitMonitoredConfigTasks( // log dir will always be with values, but might not used by the component logDir := spec.Abs(globalOptions.User, monitoredOptions.LogDir) // Generate configs - t := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). + // t := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). + t := task.NewBuilder(logger). + UserSSH( + host, + info.ssh, + globalOptions.User, + sshTimeout, + exeTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + p.Password, + p.IdentityFile, + p.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + globalOptions.SSHType, + ). MonitoredConfig( name, comp, @@ -589,7 +677,7 @@ func buildInitConfigTasks( logDir := spec.Abs(base.User, instance.LogDir()) // Download and copy the latest component to remote if the cluster is imported from Ansible - tb := task.NewBuilder(gOpt.DisplayMode) + tb := task.NewBuilder(m.logger) if instance.IsImported() { switch compName { case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: @@ -634,6 +722,7 @@ func buildInitConfigTasks( func buildDownloadCompTasks( clusterVersion string, topo spec.Topology, + logger *logprinter.Logger, gOpt operator.Options, bindVersion spec.BindVersion, ) []*task.StepDisplay { @@ -648,12 +737,12 @@ func buildDownloadCompTasks( var version string if inst.ComponentName() == spec.ComponentTiSpark { // download spark as dependency of tispark - tasks = append(tasks, buildDownloadSparkTask(inst, gOpt)) + tasks = append(tasks, buildDownloadSparkTask(inst, logger, gOpt)) } else { version = bindVersion(inst.ComponentName(), clusterVersion) } - t := task.NewBuilder(gOpt.DisplayMode). + t := task.NewBuilder(logger). Download(inst.ComponentName(), inst.OS(), inst.Arch(), version). BuildAsStep(fmt.Sprintf(" - Download %s:%s (%s/%s)", inst.ComponentName(), version, inst.OS(), inst.Arch())) @@ -665,8 +754,8 @@ func buildDownloadCompTasks( // buildDownloadSparkTask build download task for spark, which is a dependency of tispark // FIXME: this is a hack and should be replaced by dependency handling in manifest processing -func buildDownloadSparkTask(inst spec.Instance, gOpt operator.Options) *task.StepDisplay { - return task.NewBuilder(gOpt.DisplayMode). +func buildDownloadSparkTask(inst spec.Instance, logger *logprinter.Logger, gOpt operator.Options) *task.StepDisplay { + return task.NewBuilder(logger). Download(spec.ComponentSpark, inst.OS(), inst.Arch(), ""). BuildAsStep(fmt.Sprintf(" - Download %s: (%s/%s)", spec.ComponentSpark, inst.OS(), inst.Arch())) diff --git a/pkg/cluster/manager/check.go b/pkg/cluster/manager/check.go index a48b0f837d..47ba547b28 100644 --- a/pkg/cluster/manager/check.go +++ b/pkg/cluster/manager/check.go @@ -29,7 +29,7 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" ) @@ -47,6 +47,12 @@ type CheckOptions struct { // CheckCluster check cluster before deploying or upgrading func (m *Manager) CheckCluster(clusterOrTopoName string, opt CheckOptions, gOpt operator.Options) error { var topo spec.Specification + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if opt.ExistCluster { // check for existing cluster clusterName := clusterOrTopoName @@ -109,7 +115,7 @@ func (m *Manager) CheckCluster(clusterOrTopoName string, opt CheckOptions, gOpt return err } - if err := checkSystemInfo(sshConnProps, sshProxyProps, &topo, &gOpt, &opt); err != nil { + if err := checkSystemInfo(ctx, sshConnProps, sshProxyProps, &topo, &gOpt, &opt); err != nil { return err } @@ -123,7 +129,13 @@ func (m *Manager) CheckCluster(clusterOrTopoName string, opt CheckOptions, gOpt } // checkSystemInfo performs series of checks and tests of the deploy server -func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOpt *operator.Options, opt *CheckOptions) error { +func checkSystemInfo( + ctx context.Context, + s, p *tui.SSHConnectionProps, + topo *spec.Specification, + gOpt *operator.Options, + opt *CheckOptions, +) error { var ( collectTasks []*task.StepDisplay checkSysTasks []*task.StepDisplay @@ -131,6 +143,7 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp applyFixTasks []*task.StepDisplay downloadTasks []*task.StepDisplay ) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) insightVer := spec.TiDBComponentVersion(spec.ComponentCheckCollector, "") uniqueHosts := map[string]int{} // host -> ssh-port @@ -151,7 +164,7 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp archKey := fmt.Sprintf("%s-%s", inst.OS(), inst.Arch()) if _, found := uniqueArchList[archKey]; !found { uniqueArchList[archKey] = struct{}{} - t0 := task.NewBuilder(gOpt.DisplayMode). + t0 := task.NewBuilder(logger). Download( spec.ComponentCheckCollector, inst.OS(), @@ -162,7 +175,7 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp downloadTasks = append(downloadTasks, t0) } - t1 := task.NewBuilder(gOpt.DisplayMode) + t1 := task.NewBuilder(logger) // checks that applies to each instance if opt.ExistCluster { t1 = t1.CheckSys( @@ -201,7 +214,7 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp if _, found := uniqueHosts[inst.GetHost()]; !found { uniqueHosts[inst.GetHost()] = inst.GetSSHPort() // build system info collecting tasks - t2 := task.NewBuilder(gOpt.DisplayMode). + t2 := task.NewBuilder(logger). RootSSH( inst.GetHost(), inst.GetSSHPort(), @@ -322,7 +335,7 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp t1.BuildAsStep(fmt.Sprintf(" - Checking node %s", inst.GetHost())), ) - t3 := task.NewBuilder(gOpt.DisplayMode). + t3 := task.NewBuilder(logger). RootSSH( inst.GetHost(), inst.GetSSHPort(), @@ -348,14 +361,13 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp } } - t := task.NewBuilder(gOpt.DisplayMode). + t := task.NewBuilder(logger). ParallelStep("+ Download necessary tools", false, downloadTasks...). ParallelStep("+ Collect basic system information", false, collectTasks...). ParallelStep("+ Check system requirements", false, checkSysTasks...). ParallelStep("+ Cleanup check files", false, cleanTasks...). Build() - ctx := ctxt.New(context.Background(), gOpt.Concurrency) if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. @@ -371,7 +383,7 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp } checkResults := make([]HostCheckResult, 0) for host := range uniqueHosts { - tf := task.NewBuilder(gOpt.DisplayMode). + tf := task.NewBuilder(logger). RootSSH( host, uniqueHosts[host], @@ -406,7 +418,7 @@ func checkSystemInfo(s, p *tui.SSHConnectionProps, topo *spec.Specification, gOp tui.PrintTable(checkResultTable, true) if opt.ApplyFix { - tc := task.NewBuilder(gOpt.DisplayMode). + tc := task.NewBuilder(logger). ParallelStep("+ Try to apply changes to fix failed checks", false, applyFixTasks...). Build() if err := tc.Execute(ctx); err != nil { @@ -441,7 +453,7 @@ func handleCheckResults(ctx context.Context, host string, opt *CheckOptions, t * } items := make([]HostCheckResult, 0) - // log.Infof("Check results of %s: (only errors and important info are displayed)", color.HiCyanString(host)) + // m.logger.Infof("Check results of %s: (only errors and important info are displayed)", color.HiCyanString(host)) for _, r := range results { var item HostCheckResult if r.Err != nil { @@ -456,7 +468,8 @@ func handleCheckResults(ctx context.Context, host string, opt *CheckOptions, t * } msg, err := fixFailedChecks(host, r, t) if err != nil { - log.Debugf("%s: fail to apply fix to %s (%s)", host, r.Name, err) + ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger). + Debugf("%s: fail to apply fix to %s (%s)", host, r.Name, err) } if msg != "" { // show auto fixing info @@ -546,13 +559,14 @@ func fixFailedChecks(host string, res *operator.CheckResult, t *task.Builder) (s // checkRegionsInfo checks peer status from PD func (m *Manager) checkRegionsInfo(clusterName string, topo *spec.Specification, gOpt *operator.Options) error { - log.Infof("Checking region status of the cluster %s...", clusterName) + m.logger.Infof("Checking region status of the cluster %s...", clusterName) tlsConfig, err := topo.TLSConfig(m.specManager.Path(clusterName, spec.TLSCertKeyDir)) if err != nil { return err } pdClient := api.NewPDClient( + context.WithValue(context.TODO(), logprinter.ContextKeyLogger, m.logger), topo.GetPDList(), time.Second*time.Duration(gOpt.APITimeout), tlsConfig, @@ -568,7 +582,7 @@ func (m *Manager) checkRegionsInfo(clusterName string, topo *spec.Specification, return err } if rInfo.Count > 0 { - log.Warnf( + m.logger.Warnf( "Regions are not fully healthy: %s", color.YellowString("%d %s", rInfo.Count, state), ) @@ -576,9 +590,9 @@ func (m *Manager) checkRegionsInfo(clusterName string, topo *spec.Specification, } } if hasUnhealthy { - log.Warnf("Please fix unhealthy regions before other operations.") + m.logger.Warnf("Please fix unhealthy regions before other operations.") } else { - log.Infof("All regions are healthy.") + m.logger.Infof("All regions are healthy.") } return nil } diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index 861e0e7488..ae077fc24b 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" ) @@ -60,12 +60,12 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.CleanupAuditLog, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes) if !skipConfirm { - if err := cleanupConfirm(name, m.sysName, base.Version, cleanOpt, delFileMap); err != nil { + if err := cleanupConfirm(m.logger, name, m.sysName, base.Version, cleanOpt, delFileMap); err != nil { return err } } - log.Infof("Cleanup cluster...") + m.logger.Infof("Cleanup cluster...") b, err := m.sshTaskBuilder(name, topo, base.User, gOpt) if err != nil { @@ -80,7 +80,12 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper }). Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -88,13 +93,13 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper return perrs.Trace(err) } - log.Infof("Cleanup%s in cluster `%s` successfully", cleanTarget(cleanOpt), name) + m.logger.Infof("Cleanup%s in cluster `%s` successfully", cleanTarget(cleanOpt), name) return nil } // checkConfirm -func cleanupConfirm(clusterName, sysName, version string, cleanOpt operator.Options, delFileMap map[string]set.StringSet) error { - log.Warnf("The clean operation will %s %s %s cluster `%s`", +func cleanupConfirm(logger *logprinter.Logger, clusterName, sysName, version string, cleanOpt operator.Options, delFileMap map[string]set.StringSet) error { + logger.Warnf("The clean operation will %s %s %s cluster `%s`", color.HiYellowString("stop"), sysName, version, color.HiYellowString(clusterName)) if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { return err @@ -114,7 +119,7 @@ func cleanupConfirm(clusterName, sysName, version string, cleanOpt operator.Opti } } - log.Warnf("Clean the clutser %s's%s.\nNodes will be ignored: %s\nRoles will be ignored: %s\nFiles to be deleted are: %s", + logger.Warnf("Clean the clutser %s's%s.\nNodes will be ignored: %s\nRoles will be ignored: %s\nFiles to be deleted are: %s", color.HiYellowString(clusterName), cleanTarget(cleanOpt), cleanOpt.RetainDataNodes, cleanOpt.RetainDataRoles, delFileList) diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index c3fcbf8a75..aca61d864f 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -31,7 +31,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/environment" - "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -225,10 +225,11 @@ func (m *Manager) Deploy( if strings.HasPrefix(globalOptions.DataDir, "/") { dirs = append(dirs, globalOptions.DataDir) } - t := task.NewBuilder(gOpt.DisplayMode). + + t := task.NewBuilder(m.logger). RootSSH( - host, - hostInfo.ssh, + inst.GetHost(), + inst.GetSSHPort(), opt.User, sshConnProps.Password, sshConnProps.IdentityFile, @@ -245,9 +246,9 @@ func (m *Manager) Deploy( gOpt.SSHType, globalOptions.SSHType, ). - EnvInit(host, globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). - Mkdir(globalOptions.User, host, dirs...). - BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", host, hostInfo.ssh)) + EnvInit(inst.GetHost(), globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). + Mkdir(globalOptions.User, inst.GetHost(), dirs...). + BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", inst.GetHost(), inst.GetSSHPort())) envInitTasks = append(envInitTasks, t) } @@ -256,7 +257,7 @@ func (m *Manager) Deploy( } // Download missing component - downloadCompTasks = buildDownloadCompTasks(clusterVersion, topo, gOpt, m.bindVersion) + downloadCompTasks = buildDownloadCompTasks(clusterVersion, topo, m.logger, gOpt, m.bindVersion) // Deploy components to remote topo.IterInstance(func(inst spec.Instance) { @@ -275,7 +276,25 @@ func (m *Manager) Deploy( filepath.Join(deployDir, "scripts"), } - t := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), globalOptions.User, gOpt, sshProxyProps, globalOptions.SSHType). + //t := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), globalOptions.User, gOpt, sshProxyProps, globalOptions.SSHType). + + t := task.NewBuilder(m.logger). + UserSSH( + inst.GetHost(), + inst.GetSSHPort(), + globalOptions.User, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + sshProxyProps.Password, + sshProxyProps.IdentityFile, + sshProxyProps.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + globalOptions.SSHType, + ). Mkdir(globalOptions.User, inst.GetHost(), deployDirs...). Mkdir(globalOptions.User, inst.GetHost(), dataDirs...) @@ -371,9 +390,14 @@ func (m *Manager) Deploy( sshProxyProps, ) - builder := task.NewBuilder(gOpt.DisplayMode). + // builder := task.NewBuilder(gOpt.DisplayMode). + + builder := task.NewBuilder(m.logger). Step("+ Generate SSH keys", - task.NewBuilder(gOpt.DisplayMode).SSHKeyGen(m.specManager.Path(name, "ssh", "id_rsa")).Build()). + task.NewBuilder(m.logger). + SSHKeyGen(m.specManager.Path(name, "ssh", "id_rsa")). + Build(), + m.logger). ParallelStep("+ Download TiDB components", false, downloadCompTasks...). ParallelStep("+ Initialize target host environments", false, envInitTasks...). ParallelStep("+ Deploy TiDB instance", false, deployCompTasks...). @@ -387,7 +411,12 @@ func (m *Manager) Deploy( t := builder.Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -402,6 +431,6 @@ func (m *Manager) Deploy( } hint := color.New(color.Bold).Sprintf("%s start %s", tui.OsArgs0(), name) - log.Infof("Cluster `%s` deployed successfully, you can start it with command: `%s`", name, hint) + m.logger.Infof("Cluster `%s` deployed successfully, you can start it with command: `%s`", name, hint) return nil } diff --git a/pkg/cluster/manager/destroy.go b/pkg/cluster/manager/destroy.go index 57f6d108a2..c1a0729ecf 100644 --- a/pkg/cluster/manager/destroy.go +++ b/pkg/cluster/manager/destroy.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/tui" ) @@ -60,7 +59,7 @@ func (m *Manager) DestroyCluster(name string, gOpt operator.Options, destroyOpt color.HiYellowString(name)); err != nil { return err } - log.Infof("Destroying cluster...") + m.logger.Infof("Destroying cluster...") } b, err := m.sshTaskBuilder(name, topo, base.User, gOpt) @@ -76,7 +75,12 @@ func (m *Manager) DestroyCluster(name string, gOpt operator.Options, destroyOpt }). Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -88,7 +92,7 @@ func (m *Manager) DestroyCluster(name string, gOpt operator.Options, destroyOpt return perrs.Trace(err) } - log.Infof("Destroyed cluster `%s` successfully", name) + m.logger.Infof("Destroyed cluster `%s` successfully", name) return nil } @@ -126,7 +130,11 @@ func (m *Manager) DestroyTombstone( return err } - ctx := ctxt.New(context.Background(), gOpt.Concurrency) + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) nodes, err := operator.DestroyTombstone(ctx, cluster, true /* returnNodesOnly */, gOpt, tlsCfg) if err != nil { return err @@ -145,14 +153,14 @@ func (m *Manager) DestroyTombstone( return err } } - log.Infof("Start destroy Tombstone nodes: %v ...", nodes) + m.logger.Infof("Start destroy Tombstone nodes: %v ...", nodes) return err }). ClusterOperate(cluster, operator.DestroyTombstoneOperation, gOpt, tlsCfg). UpdateMeta(name, clusterMeta, nodes). UpdateTopology(name, m.specManager.Path(name), clusterMeta, nodes). ParallelStep("+ Refresh instance configs", true, regenConfigTasks...). - Parallel(true, buildReloadPromTasks(metadata.GetTopology(), gOpt)...). + Parallel(true, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...). Build() if err := t.Execute(ctx); err != nil { @@ -163,7 +171,7 @@ func (m *Manager) DestroyTombstone( return perrs.Trace(err) } - log.Infof("Destroy success") + m.logger.Infof("Destroy success") return nil } diff --git a/pkg/cluster/manager/display.go b/pkg/cluster/manager/display.go index 44d01a7b8a..5ebac152c4 100644 --- a/pkg/cluster/manager/display.go +++ b/pkg/cluster/manager/display.go @@ -15,10 +15,12 @@ package manager import ( "context" + "crypto/tls" "encoding/json" "errors" "fmt" "math" + "net/url" "sort" "strconv" "strings" @@ -32,11 +34,13 @@ import ( "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/crypto" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" + "go.uber.org/zap" ) // InstInfo represents an instance info @@ -107,7 +111,7 @@ func (m *Manager) Display(name string, opt operator.Options) error { cyan := color.New(color.FgCyan, color.Bold) // display cluster meta var j *JSONOutput - if log.GetDisplayMode() == log.DisplayModeJSON { + if m.logger.GetDisplayMode() == logprinter.DisplayModeJSON { j = &JSONOutput{ ClusterMetaInfo: ClusterMetaInfo{ m.sysName, @@ -190,15 +194,20 @@ func (m *Manager) Display(name string, opt operator.Options) error { } var dashboardAddr string + ctx := ctxt.New( + context.Background(), + opt.Concurrency, + m.logger, + ) if t, ok := topo.(*spec.Specification); ok { var err error - dashboardAddr, err = t.GetDashboardAddress(tlsCfg, masterActive...) + dashboardAddr, err = t.GetDashboardAddress(ctx, tlsCfg, masterActive...) if err == nil && !set.NewStringSet("", "auto", "none").Exist(dashboardAddr) { scheme := "http" if tlsCfg != nil { scheme = "https" } - if log.GetDisplayMode() == log.DisplayModeJSON { + if m.logger.GetDisplayMode() == logprinter.DisplayModeJSON { j.ClusterMetaInfo.DashboardURL = fmt.Sprintf("%s://%s/dashboard", scheme, dashboardAddr) } else { fmt.Printf("Dashboard URL: %s\n", cyan.Sprintf("%s://%s/dashboard", scheme, dashboardAddr)) @@ -206,7 +215,7 @@ func (m *Manager) Display(name string, opt operator.Options) error { } } - if log.GetDisplayMode() == log.DisplayModeJSON { + if m.logger.GetDisplayMode() == logprinter.DisplayModeJSON { d, err := json.MarshalIndent(j, "", " ") if err != nil { return err @@ -218,13 +227,17 @@ func (m *Manager) Display(name string, opt operator.Options) error { tui.PrintTable(clusterTable, true) fmt.Printf("Total nodes: %d\n", len(clusterTable)-1) - ctx := ctxt.New(context.Background(), opt.Concurrency) if t, ok := topo.(*spec.Specification); ok { // Check if TiKV's label set correctly - pdClient := api.NewPDClient(masterActive, 10*time.Second, tlsCfg) + pdClient := api.NewPDClient( + context.WithValue(ctx, logprinter.ContextKeyLogger, m.logger), + masterActive, + 10*time.Second, + tlsCfg, + ) if lbs, placementRule, err := pdClient.GetLocationLabels(); err != nil { - log.Debugf("get location labels from pd failed: %v", err) + m.logger.Debugf("get location labels from pd failed: %v", err) } else if !placementRule { if err := spec.CheckTiKVLabels(lbs, pdClient); err != nil { color.Yellow("\nWARN: there is something wrong with TiKV labels, which may cause data losing:\n%v", err) @@ -314,6 +327,12 @@ func (m *Manager) DisplayTiKVLabels(name string, opt operator.Options) error { } } + ctx := ctxt.New( + context.Background(), + opt.Concurrency, + m.logger, + ) + masterList := topo.BaseTopo().MasterList tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) if err != nil { @@ -321,7 +340,7 @@ func (m *Manager) DisplayTiKVLabels(name string, opt operator.Options) error { } topo.IterInstance(func(ins spec.Instance) { if ins.ComponentName() == spec.ComponentPD { - status := ins.Status(tlsCfg, masterList...) + status := ins.Status(ctx, tlsCfg, masterList...) if strings.HasPrefix(status, "Up") || strings.HasPrefix(status, "Healthy") { instAddr := fmt.Sprintf("%s:%d", ins.GetHost(), ins.GetPort()) masterActive = append(masterActive, instAddr) @@ -336,16 +355,16 @@ func (m *Manager) DisplayTiKVLabels(name string, opt operator.Options) error { if _, ok := topo.(*spec.Specification); ok { // Check if TiKV's label set correctly - pdClient := api.NewPDClient(masterActive, 10*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, masterActive, 10*time.Second, tlsCfg) // No locationLabel, _, err = pdClient.GetLocationLabels() if err != nil { - log.Debugf("get location labels from pd failed: %v", err) + m.logger.Debugf("get location labels from pd failed: %v", err) } _, storeInfos, err := pdClient.GetTiKVLabels() if err != nil { - log.Debugf("get tikv state and labels from pd failed: %v", err) + m.logger.Debugf("get tikv state and labels from pd failed: %v", err) } for storeIP := range tikvStoreIP { @@ -402,7 +421,11 @@ func (m *Manager) DisplayTiKVLabels(name string, opt operator.Options) error { // GetClusterTopology get the topology of the cluster. func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstInfo, error) { - ctx := ctxt.New(context.Background(), opt.Concurrency) + ctx := ctxt.New( + context.Background(), + opt.Concurrency, + m.logger, + ) metadata, err := m.meta(name) if err != nil && !errors.Is(perrs.Cause(err), meta.ErrValidate) && !errors.Is(perrs.Cause(err), spec.ErrNoTiSparkMaster) { @@ -437,7 +460,7 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI if ins.ComponentName() != spec.ComponentPD && ins.ComponentName() != spec.ComponentDMMaster { return } - status := ins.Status(tlsCfg, masterList...) + status := ins.Status(ctx, tlsCfg, masterList...) if strings.HasPrefix(status, "Up") || strings.HasPrefix(status, "Healthy") { instAddr := fmt.Sprintf("%s:%d", ins.GetHost(), ins.GetPort()) masterActive = append(masterActive, instAddr) @@ -447,7 +470,7 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI var dashboardAddr string if t, ok := topo.(*spec.Specification); ok { - dashboardAddr, _ = t.GetDashboardAddress(tlsCfg, masterActive...) + dashboardAddr, _ = t.GetDashboardAddress(ctx, tlsCfg, masterActive...) } clusterInstInfos := []InstInfo{} @@ -480,12 +503,12 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI case spec.ComponentDMMaster: status = masterStatus[ins.ID()] default: - status = ins.Status(tlsCfg, masterActive...) + status = ins.Status(ctx, tlsCfg, masterActive...) } since := "-" if opt.ShowUptime { - since = formatInstanceSince(ins.Uptime(tlsCfg)) + since = formatInstanceSince(ins.Uptime(ctx, tlsCfg)) } // Query the service status and uptime @@ -612,7 +635,7 @@ func parseSystemctlSince(str string) (dur time.Duration) { } defer func() { if dur == 0 { - log.Warnf("failed to parse systemctl since '%s'", str) + zap.L().Warn("failed to parse systemctl since", zap.String("value", str)) } }() parts := strings.Split(str, ";") @@ -670,3 +693,53 @@ func SetClusterSSH(ctx context.Context, topo spec.Topology, deployUser string, s return nil } + +// DisplayDashboardInfo prints the dashboard address of cluster +func (m *Manager) DisplayDashboardInfo(clusterName string, tlsCfg *tls.Config) error { + metadata, err := spec.ClusterMetadata(clusterName) + if err != nil && !errors.Is(perrs.Cause(err), meta.ErrValidate) && + !errors.Is(perrs.Cause(err), spec.ErrNoTiSparkMaster) { + return err + } + + pdEndpoints := make([]string, 0) + for _, pd := range metadata.Topology.PDServers { + pdEndpoints = append(pdEndpoints, fmt.Sprintf("%s:%d", pd.Host, pd.ClientPort)) + } + + ctx := context.WithValue(context.Background(), logprinter.ContextKeyLogger, m.logger) + pdAPI := api.NewPDClient(ctx, pdEndpoints, 2*time.Second, tlsCfg) + dashboardAddr, err := pdAPI.GetDashboardAddress() + if err != nil { + return fmt.Errorf("failed to retrieve TiDB Dashboard instance from PD: %s", err) + } + if dashboardAddr == "auto" { + return fmt.Errorf("TiDB Dashboard is not initialized, please start PD and try again") + } else if dashboardAddr == "none" { + return fmt.Errorf("TiDB Dashboard is disabled") + } + + u, err := url.Parse(dashboardAddr) + if err != nil { + return fmt.Errorf("unknown TiDB Dashboard PD instance: %s", dashboardAddr) + } + + u.Path = "/dashboard/" + + if tlsCfg != nil { + fmt.Println( + "Client certificate:", + color.CyanString(m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.PFXClientCert)), + ) + fmt.Println( + "Certificate password:", + color.CyanString(crypto.PKCS12Password), + ) + } + fmt.Println( + "Dashboard URL:", + color.CyanString(u.String()), + ) + + return nil +} diff --git a/pkg/cluster/manager/edit_config.go b/pkg/cluster/manager/edit_config.go index 32486f0b8e..d4a2746858 100644 --- a/pkg/cluster/manager/edit_config.go +++ b/pkg/cluster/manager/edit_config.go @@ -24,7 +24,6 @@ import ( perrs "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" @@ -63,14 +62,14 @@ func (m *Manager) EditConfig(name string, opt EditConfigOptions, skipConfirm boo return nil } - log.Infof("Applying changes...") + m.logger.Infof("Applying changes...") metadata.SetTopology(newTopo) err = m.specManager.SaveMeta(name, metadata) if err != nil { return perrs.Annotate(err, "failed to save meta") } - log.Infof("Applied successfully, please use `%s reload %s [-N ] [-R ]` to reload config.", tui.OsArgs0(), name) + m.logger.Infof("Applied successfully, please use `%s reload %s [-N ] [-R ]` to reload config.", tui.OsArgs0(), name) return nil } @@ -117,26 +116,26 @@ func (m *Manager) editTopo(origTopo spec.Topology, data []byte, opt EditConfigOp err = yaml.UnmarshalStrict(newData, newTopo) if err != nil { fmt.Print(color.RedString("New topology could not be saved: ")) - log.Infof("Failed to parse topology file: %v", err) + m.logger.Infof("Failed to parse topology file: %v", err) if opt.NewTopoFile == "" { if pass, _ := tui.PromptForConfirmNo("Do you want to continue editing? [Y/n]: "); !pass { return m.editTopo(origTopo, newData, opt, skipConfirm) } } - log.Infof("Nothing changed.") + m.logger.Infof("Nothing changed.") return nil, nil } // report error if immutable field has been changed if err := utils.ValidateSpecDiff(origTopo, newTopo); err != nil { fmt.Print(color.RedString("New topology could not be saved: ")) - log.Errorf("%s", err) + m.logger.Errorf("%s", err) if opt.NewTopoFile == "" { if pass, _ := tui.PromptForConfirmNo("Do you want to continue editing? [Y/n]: "); !pass { return m.editTopo(origTopo, newData, opt, skipConfirm) } } - log.Infof("Nothing changed.") + m.logger.Infof("Nothing changed.") return nil, nil } @@ -146,7 +145,7 @@ func (m *Manager) editTopo(origTopo spec.Topology, data []byte, opt EditConfigOp } if bytes.Equal(origData, newData) { - log.Infof("The file has nothing changed") + m.logger.Infof("The file has nothing changed") return nil, nil } diff --git a/pkg/cluster/manager/exec.go b/pkg/cluster/manager/exec.go index d8a667331f..1d67d46301 100644 --- a/pkg/cluster/manager/exec.go +++ b/pkg/cluster/manager/exec.go @@ -26,7 +26,6 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/set" ) @@ -68,7 +67,7 @@ func (m *Manager) Exec(name string, opt ExecOptions, gOpt operator.Options) erro cmds, err := renderInstanceSpec(opt.Command, inst) if err != nil { - log.Debugf("error rendering command with spec: %s", err) + m.logger.Debugf("error rendering command with spec: %s", err) return // skip } cmdSet := set.NewStringSet(cmds...) @@ -84,7 +83,7 @@ func (m *Manager) Exec(name string, opt ExecOptions, gOpt operator.Options) erro host := strings.Split(hostKey, "-")[0] for _, cmd := range i.Slice() { shellTasks = append(shellTasks, - task.NewBuilder(gOpt.DisplayMode). + task.NewBuilder(m.logger). Shell(host, cmd, hostKey+cmd, opt.Sudo). Build()) } @@ -99,7 +98,11 @@ func (m *Manager) Exec(name string, opt ExecOptions, gOpt operator.Options) erro Parallel(false, shellTasks...). Build() - execCtx := ctxt.New(context.Background(), gOpt.Concurrency) + execCtx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) if err := t.Execute(execCtx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. @@ -116,14 +119,14 @@ func (m *Manager) Exec(name string, opt ExecOptions, gOpt operator.Options) erro if !ok { continue } - log.Infof("Outputs of %s on %s:", + m.logger.Infof("Outputs of %s on %s:", color.CyanString(cmd), color.CyanString(host)) if len(stdout) > 0 { - log.Infof("%s:\n%s", color.GreenString("stdout"), stdout) + m.logger.Infof("%s:\n%s", color.GreenString("stdout"), stdout) } if len(stderr) > 0 { - log.Infof("%s:\n%s", color.RedString("stderr"), stderr) + m.logger.Infof("%s:\n%s", color.RedString("stderr"), stderr) } } } diff --git a/pkg/cluster/manager/list.go b/pkg/cluster/manager/list.go index 41e0182525..c0bf613da0 100644 --- a/pkg/cluster/manager/list.go +++ b/pkg/cluster/manager/list.go @@ -20,7 +20,7 @@ import ( perrs "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/tui" ) @@ -41,8 +41,8 @@ func (m *Manager) ListCluster() error { return err } - switch log.GetDisplayMode() { - case log.DisplayModeJSON: + switch m.logger.GetDisplayMode() { + case logprinter.DisplayModeJSON: clusterObj := struct { Clusters []Cluster `json:"clusters"` }{ diff --git a/pkg/cluster/manager/manager.go b/pkg/cluster/manager/manager.go index 2626b69329..569ac05ce4 100644 --- a/pkg/cluster/manager/manager.go +++ b/pkg/cluster/manager/manager.go @@ -26,7 +26,7 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" @@ -46,14 +46,21 @@ type Manager struct { sysName string specManager *spec.SpecManager bindVersion spec.BindVersion + logger *logprinter.Logger } // NewManager create a Manager. -func NewManager(sysName string, specManager *spec.SpecManager, bindVersion spec.BindVersion) *Manager { +func NewManager( + sysName string, + specManager *spec.SpecManager, + bindVersion spec.BindVersion, + logger *logprinter.Logger, +) *Manager { return &Manager{ sysName: sysName, specManager: specManager, bindVersion: bindVersion, + logger: logger, } } @@ -77,7 +84,7 @@ func (m *Manager) meta(name string) (metadata spec.Metadata, err error) { } func (m *Manager) confirmTopology(name, version string, topo spec.Topology, patchedRoles set.StringSet) error { - log.Infof("Please confirm your topology:") + m.logger.Infof("Please confirm your topology:") cyan := color.New(color.FgCyan, color.Bold) fmt.Printf("Cluster type: %s\n", cyan.Sprint(m.sysName)) @@ -108,11 +115,11 @@ func (m *Manager) confirmTopology(name, version string, topo spec.Topology, patc tui.PrintTable(clusterTable, true) - log.Warnf("Attention:") - log.Warnf(" 1. If the topology is not what you expected, check your yaml file.") - log.Warnf(" 2. Please confirm there is no port/directory conflicts in same host.") + m.logger.Warnf("Attention:") + m.logger.Warnf(" 1. If the topology is not what you expected, check your yaml file.") + m.logger.Warnf(" 2. Please confirm there is no port/directory conflicts in same host.") if len(patchedRoles) != 0 { - log.Errorf(" 3. The component marked as `patched` has been replaced by previous patch commanm.") + m.logger.Errorf(" 3. The component marked as `patched` has been replaced by previous patch commanm.") } if spec, ok := topo.(*spec.Specification); ok { @@ -121,7 +128,7 @@ func (m *Manager) confirmTopology(name, version string, topo spec.Topology, patc msg := cyan.Sprint(`There are TiSpark nodes defined in the topology, please note that you'll need to manually install Java Runtime Environment (JRE) 8 on the host, otherwise the TiSpark nodes will fail to start. You may read the OpenJDK doc for a reference: https://openjdk.java.net/install/ `) - log.Warnf(msg) + m.logger.Warnf(msg) } } @@ -137,7 +144,7 @@ func (m *Manager) sshTaskBuilder(name string, topo spec.Topology, user string, g } } - return task.NewBuilder(gOpt.DisplayMode). + return task.NewBuilder(m.logger). SSHKeySet( m.specManager.Path(name, "ssh", "id_rsa"), m.specManager.Path(name, "ssh", "id_rsa.pub"), @@ -172,7 +179,7 @@ func (m *Manager) fillHostArch(s, p *tui.SSHConnectionProps, topo spec.Topology, } hostArch[inst.GetHost()] = "" - tf := task.NewBuilder(gOpt.DisplayMode). + tf := task.NewBuilder(m.logger). RootSSH( inst.GetHost(), inst.GetSSHPort(), @@ -200,8 +207,12 @@ func (m *Manager) fillHostArch(s, p *tui.SSHConnectionProps, topo spec.Topology, return nil } - ctx := ctxt.New(context.Background(), gOpt.Concurrency) - t := task.NewBuilder(gOpt.DisplayMode). + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + t := task.NewBuilder(m.logger). ParallelStep("+ Detect CPU Arch", false, detectTasks...). Build() diff --git a/pkg/cluster/manager/patch.go b/pkg/cluster/manager/patch.go index 8e51f36ce1..47af39e309 100644 --- a/pkg/cluster/manager/patch.go +++ b/pkg/cluster/manager/patch.go @@ -85,7 +85,7 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o var replacePackageTasks []task.Task for _, inst := range insts { deployDir := spec.Abs(base.User, inst.DeployDir()) - tb := task.NewBuilder(opt.DisplayMode) + tb := task.NewBuilder(m.logger) tb.BackupComponent(inst.ComponentName(), base.Version, inst.GetHost(), deployDir). InstallPackage(packagePath, inst.GetHost(), deployDir) replacePackageTasks = append(replacePackageTasks, tb.Build()) @@ -108,7 +108,12 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o }). Build() - if err := t.Execute(ctxt.New(context.Background(), opt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + opt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err diff --git a/pkg/cluster/manager/reload.go b/pkg/cluster/manager/reload.go index 5f7602d2a3..e75fcc976d 100644 --- a/pkg/cluster/manager/reload.go +++ b/pkg/cluster/manager/reload.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/tui" ) @@ -93,6 +93,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo noAgentHosts, *topo.BaseTopo().GlobalOptions, topo.GetMonitoredOptions(), + m.logger, sshTimeout, exeTimeout, gOpt, @@ -129,7 +130,12 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo t := b.Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -137,7 +143,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo return perrs.Trace(err) } - log.Infof("Reloaded cluster `%s` successfully", name) + m.logger.Infof("Reloaded cluster `%s` successfully", name) return nil } diff --git a/pkg/cluster/manager/rename.go b/pkg/cluster/manager/rename.go index b27fb3d1c4..e7237170c1 100644 --- a/pkg/cluster/manager/rename.go +++ b/pkg/cluster/manager/rename.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tiup/pkg/cluster/clusterutil" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" ) @@ -63,7 +62,7 @@ func (m *Manager) Rename(name string, opt operator.Options, newName string, skip return err } - log.Infof("Rename cluster `%s` -> `%s` successfully", name, newName) + m.logger.Infof("Rename cluster `%s` -> `%s` successfully", name, newName) opt.Roles = []string{spec.ComponentGrafana, spec.ComponentPrometheus} return m.Reload(newName, opt, false, skipConfirm) diff --git a/pkg/cluster/manager/scale_in.go b/pkg/cluster/manager/scale_in.go index 44b4b131e8..e575629ca3 100644 --- a/pkg/cluster/manager/scale_in.go +++ b/pkg/cluster/manager/scale_in.go @@ -27,7 +27,6 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/tui" ) @@ -54,7 +53,7 @@ func (m *Manager) ScaleIn( ) if !skipConfirm { if force { - log.Warnf(color.HiRedString(tui.ASCIIArtWarning)) + m.logger.Warnf(color.HiRedString(tui.ASCIIArtWarning)) if err := tui.PromptForAnswerOrAbortError( "Yes, I know my data might be lost.", color.HiRedString("Forcing scale in is unsafe and may result in data loss for stateful components.\n"+ @@ -75,7 +74,7 @@ func (m *Manager) ScaleIn( return err } - log.Infof("Scale-in nodes...") + m.logger.Infof("Scale-in nodes...") } metadata, err := m.meta(name) @@ -117,10 +116,15 @@ func (m *Manager) ScaleIn( t := b. ParallelStep("+ Refresh instance configs", force, regenConfigTasks...). - Parallel(force, buildReloadPromTasks(metadata.GetTopology(), gOpt, nodes...)...). + Parallel(force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...). Build() - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -128,7 +132,7 @@ func (m *Manager) ScaleIn( return perrs.Trace(err) } - log.Infof("Scaled cluster `%s` in successfully", name) + m.logger.Infof("Scaled cluster `%s` in successfully", name) return nil } diff --git a/pkg/cluster/manager/scale_out.go b/pkg/cluster/manager/scale_out.go index 23dd411e73..7058a04de6 100644 --- a/pkg/cluster/manager/scale_out.go +++ b/pkg/cluster/manager/scale_out.go @@ -29,7 +29,7 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" @@ -81,7 +81,7 @@ func (m *Manager) ScaleOut( } else { // if stage2 is true, not need check topology or other // check for the input topology to let user confirm if there're any // global configs set - if err := checkForGlobalConfigs(topoFile, skipConfirm); err != nil { + if err := checkForGlobalConfigs(m.logger, topoFile, skipConfirm); err != nil { return err } @@ -150,7 +150,10 @@ func (m *Manager) ScaleOut( if err != nil { return err } - pdClient := api.NewPDClient(pdList, 10*time.Second, tlsCfg) + pdClient := api.NewPDClient( + context.WithValue(context.TODO(), logprinter.ContextKeyLogger, m.logger), + pdList, 10*time.Second, tlsCfg, + ) lbs, placementRule, err := pdClient.GetLocationLabels() if err != nil { return err @@ -199,7 +202,11 @@ func (m *Manager) ScaleOut( return err } - ctx := ctxt.New(context.Background(), gOpt.Concurrency) + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) ctx = context.WithValue(ctx, ctxt.CtxBaseTopo, topo) if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { @@ -210,11 +217,11 @@ func (m *Manager) ScaleOut( } if opt.Stage1 { - log.Infof(`The new instance is not started! + m.logger.Infof(`The new instance is not started! You need to execute '%s' to start the new instance.`, color.YellowString("tiup cluster scale-out %s --stage2", name)) } - log.Infof("Scaled cluster `%s` out successfully", color.YellowString(name)) + m.logger.Infof("Scaled cluster `%s` out successfully", color.YellowString(name)) return nil } @@ -236,7 +243,7 @@ func validateNewTopo(topo spec.Topology) (err error) { // checkForGlobalConfigs checks the input scale out topology to make sure users are aware // of the global config fields in it will be ignored. -func checkForGlobalConfigs(topoFile string, skipConfirm bool) error { +func checkForGlobalConfigs(logger *logprinter.Logger, topoFile string, skipConfirm bool) error { yamlFile, err := spec.ReadYamlFile(topoFile) if err != nil { return err @@ -254,7 +261,7 @@ func checkForGlobalConfigs(topoFile string, skipConfirm bool) error { case "global", "monitored", "server_configs": - log.Warnf(`You have one or more of %s fields configured in + logger.Warnf(`You have one or more of %s fields configured in the scale out topology, but they will be ignored during the scaling out process. If you want to use configs different from the existing cluster, cancel now and set them in the specification fileds for each host.`, color.YellowString(`["global", "monitored", "server_configs"]`)) @@ -282,7 +289,7 @@ func checkScaleOutLock(m *Manager, name string, opt DeployOptions, skipConfirm b return m.specManager.ScaleOutLockedErr(name) } - log.Warnf(`The parameter '%s' is set, new instance will not be started + m.logger.Warnf(`The parameter '%s' is set, new instance will not be started Please manually execute '%s' to finish the process.`, color.YellowString("--stage1"), color.YellowString("tiup cluster scale-out %s --stage2", name)) @@ -298,7 +305,7 @@ func checkScaleOutLock(m *Manager, name string, opt DeployOptions, skipConfirm b return fmt.Errorf("The scale-out file lock does not exist, please make sure to run 'tiup-cluster scale-out %s --stage1' first", name) } - log.Warnf(`The parameter '%s' is set, only start the new instances and reload configs.`, color.YellowString("--stage2")) + m.logger.Warnf(`The parameter '%s' is set, only start the new instances and reload configs.`, color.YellowString("--stage2")) if !skipConfirm { if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]: "); err != nil { return err diff --git a/pkg/cluster/manager/transfer.go b/pkg/cluster/manager/transfer.go index f28aafde78..2e316e7157 100644 --- a/pkg/cluster/manager/transfer.go +++ b/pkg/cluster/manager/transfer.go @@ -29,8 +29,8 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/set" + "go.uber.org/zap" ) // TransferOptions for exec shell commanm. @@ -77,7 +77,7 @@ func (m *Manager) Transfer(name string, opt TransferOptions, gOpt operator.Optio instPath := opt.Remote paths, err := renderInstanceSpec(instPath, inst) if err != nil { - log.Debugf("error rendering remote path with spec: %s", err) + m.logger.Debugf("error rendering remote path with spec: %s", err) return // skip } pathSet := set.NewStringSet(paths...) @@ -93,7 +93,7 @@ func (m *Manager) Transfer(name string, opt TransferOptions, gOpt operator.Optio for hostKey, i := range uniqueHosts { host := strings.Split(hostKey, "-")[0] for _, p := range i.Slice() { - t := task.NewBuilder(gOpt.DisplayMode) + t := task.NewBuilder(m.logger) if opt.Pull { t.CopyFile(p, srcPath, host, opt.Pull, opt.Limit, opt.Compress) } else { @@ -112,7 +112,11 @@ func (m *Manager) Transfer(name string, opt TransferOptions, gOpt operator.Optio Parallel(false, shellTasks...). Build() - execCtx := ctxt.New(context.Background(), gOpt.Concurrency) + execCtx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) if err := t.Execute(execCtx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. @@ -137,7 +141,7 @@ func renderInstanceSpec(t string, inst spec.Instance) ([]string, error) { key := inst.ID() + d + uuid.New().String() s, err := renderSpec(t, inst.(*spec.TiFlashInstance), key) if err != nil { - log.Debugf("error rendering tiflash spec: %s", err) + zap.L().Debug("error rendering tiflash spec", zap.Error(err)) } result = append(result, s) } @@ -173,7 +177,7 @@ func renderSpec(t string, s interface{}, id string) (string, error) { result := bytes.NewBufferString("") if err := tpl.Execute(result, s); err != nil { - log.Debugf("missing key when parsing: %s", err) + zap.L().Debug("missing key when parsing: %s", zap.Error(err)) return "", err } return result.String(), nil diff --git a/pkg/cluster/manager/upgrade.go b/pkg/cluster/manager/upgrade.go index 6bb3591368..a6e259c29d 100644 --- a/pkg/cluster/manager/upgrade.go +++ b/pkg/cluster/manager/upgrade.go @@ -27,7 +27,6 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/environment" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/tui" @@ -74,7 +73,7 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio color.HiYellowString(clusterVersion)); err != nil { return err } - log.Infof("Upgrading cluster...") + m.logger.Infof("Upgrading cluster...") } hasImported := false @@ -97,7 +96,7 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio key := fmt.Sprintf("%s-%s-%s-%s", compName, version, inst.OS(), inst.Arch()) if _, found := uniqueComps[key]; !found { uniqueComps[key] = struct{}{} - t := task.NewBuilder(opt.DisplayMode). + t := task.NewBuilder(m.logger). Download(inst.ComponentName(), inst.OS(), inst.Arch(), version). Build() downloadCompTasks = append(downloadCompTasks, t) @@ -110,7 +109,7 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio logDir := spec.Abs(base.User, inst.LogDir()) // Deploy component - tb := task.NewBuilder(opt.DisplayMode) + tb := task.NewBuilder(m.logger) // for some component, dataDirs might need to be created due to upgrade // eg: TiCDC support DataDir since v4.0.13 @@ -207,7 +206,12 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio }). Build() - if err := t.Execute(ctxt.New(context.Background(), opt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + opt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -231,7 +235,7 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio return err } - log.Infof("Upgraded cluster `%s` successfully", name) + m.logger.Infof("Upgraded cluster `%s` successfully", name) return nil } diff --git a/pkg/cluster/module/wait_for.go b/pkg/cluster/module/wait_for.go index 1aea6e2af4..8a59104e1b 100644 --- a/pkg/cluster/module/wait_for.go +++ b/pkg/cluster/module/wait_for.go @@ -21,8 +21,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/ctxt" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/utils" + "go.uber.org/zap" ) // WaitForConfig is the configurations of WaitFor module. @@ -87,7 +87,7 @@ func (w *WaitFor) Execute(ctx context.Context, e ctxt.Executor) (err error) { } return err }, retryOpt); err != nil { - log.Debugf("retry error: %s", err) + zap.L().Debug("retry error", zap.Error(err)) return errors.Errorf("timed out waiting for port %d to be %s after %s", w.c.Port, w.c.State, w.c.Timeout) } return nil diff --git a/pkg/cluster/operation/action.go b/pkg/cluster/operation/action.go index 5f4a6579a8..95f033e26d 100644 --- a/pkg/cluster/operation/action.go +++ b/pkg/cluster/operation/action.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/module" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "golang.org/x/sync/errgroup" ) @@ -266,20 +266,21 @@ func EnableMonitored(ctx context.Context, hosts []string, noAgentHosts set.Strin } func systemctlMonitor(ctx context.Context, hosts []string, noAgentHosts set.StringSet, options *spec.MonitoredOptions, action string, timeout uint64) error { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) ports := monitorPortMap(options) for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { - log.Infof("%s component %s", actionPrevMsgs[action], comp) + logger.Infof("%s component %s", actionPrevMsgs[action], comp) errg, _ := errgroup.WithContext(ctx) for _, host := range hosts { host := host if noAgentHosts.Exist(host) { - log.Debugf("Ignored %s component %s for %s", action, comp, host) + logger.Debugf("Ignored %s component %s for %s", action, comp, host) continue } nctx := checkpoint.NewContext(ctx) errg.Go(func() error { - log.Infof("\t%s instance %s", actionPrevMsgs[action], host) + logger.Infof("\t%s instance %s", actionPrevMsgs[action], host) e := ctxt.GetInner(nctx).Get(host) service := fmt.Sprintf("%s-%d.service", comp, ports[comp]) @@ -298,7 +299,7 @@ func systemctlMonitor(ctx context.Context, hosts []string, noAgentHosts set.Stri if err != nil { return toFailedActionError(err, action, host, service, "") } - log.Infof("\t%s %s success", actionPostMsgs[action], host) + logger.Infof("\t%s %s success", actionPostMsgs[action], host) return nil }) } @@ -312,7 +313,8 @@ func systemctlMonitor(ctx context.Context, hosts []string, noAgentHosts set.Stri func restartInstance(ctx context.Context, ins spec.Instance, timeout uint64, tlsCfg *tls.Config) error { e := ctxt.GetInner(ctx).Get(ins.GetHost()) - log.Infof("\tRestarting instance %s", ins.ID()) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + logger.Infof("\tRestarting instance %s", ins.ID()) if err := systemctl(ctx, e, ins.ServiceName(), "restart", timeout); err != nil { return toFailedActionError(err, "restart", ins.GetHost(), ins.ServiceName(), ins.LogDir()) @@ -323,33 +325,35 @@ func restartInstance(ctx context.Context, ins spec.Instance, timeout uint64, tls return toFailedActionError(err, "restart", ins.GetHost(), ins.ServiceName(), ins.LogDir()) } - log.Infof("\tRestart instance %s success", ins.ID()) + logger.Infof("\tRestart instance %s success", ins.ID()) return nil } func enableInstance(ctx context.Context, ins spec.Instance, timeout uint64, isEnable bool) error { e := ctxt.GetInner(ctx).Get(ins.GetHost()) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) action := "disable" if isEnable { action = "enable" } - log.Infof("\t%s instance %s", actionPrevMsgs[action], ins.ID()) + logger.Infof("\t%s instance %s", actionPrevMsgs[action], ins.ID()) // Enable/Disable by systemd. if err := systemctl(ctx, e, ins.ServiceName(), action, timeout); err != nil { return toFailedActionError(err, action, ins.GetHost(), ins.ServiceName(), ins.LogDir()) } - log.Infof("\t%s instance %s success", actionPostMsgs[action], ins.ID()) + logger.Infof("\t%s instance %s success", actionPostMsgs[action], ins.ID()) return nil } func startInstance(ctx context.Context, ins spec.Instance, timeout uint64, tlsCfg *tls.Config) error { e := ctxt.GetInner(ctx).Get(ins.GetHost()) - log.Infof("\tStarting instance %s", ins.ID()) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + logger.Infof("\tStarting instance %s", ins.ID()) if err := systemctl(ctx, e, ins.ServiceName(), "start", timeout); err != nil { return toFailedActionError(err, "start", ins.GetHost(), ins.ServiceName(), ins.LogDir()) @@ -360,12 +364,13 @@ func startInstance(ctx context.Context, ins spec.Instance, timeout uint64, tlsCf return toFailedActionError(err, "start", ins.GetHost(), ins.ServiceName(), ins.LogDir()) } - log.Infof("\tStart instance %s success", ins.ID()) + logger.Infof("\tStart instance %s success", ins.ID()) return nil } func systemctl(ctx context.Context, executor ctxt.Executor, service string, action string, timeout uint64) error { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) c := module.SystemdModuleConfig{ Unit: service, ReloadDaemon: true, @@ -379,7 +384,7 @@ func systemctl(ctx context.Context, executor ctxt.Executor, service string, acti fmt.Println(string(stdout)) } if len(stderr) > 0 && !bytes.Contains(stderr, []byte("Created symlink ")) && !bytes.Contains(stderr, []byte("Removed symlink ")) { - log.Errorf(string(stderr)) + logger.Errorf(string(stderr)) } if len(stderr) > 0 && action == "stop" { // ignore "unit not loaded" error, as this means the unit is not @@ -387,10 +392,10 @@ func systemctl(ctx context.Context, executor ctxt.Executor, service string, acti // NOTE: there will be a potential bug if the unit name is set // wrong and the real unit still remains started. if bytes.Contains(stderr, []byte(" not loaded.")) { - log.Warnf(string(stderr)) + logger.Warnf(string(stderr)) return nil // reset the error to avoid exiting } - log.Errorf(string(stderr)) + logger.Errorf(string(stderr)) } return err } @@ -401,11 +406,12 @@ func EnableComponent(ctx context.Context, instances []spec.Instance, noAgentHost return nil } + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) name := instances[0].ComponentName() if isEnable { - log.Infof("Enabling component %s", name) + logger.Infof("Enabling component %s", name) } else { - log.Infof("Disabling component %s", name) + logger.Infof("Disabling component %s", name) } errg, _ := errgroup.WithContext(ctx) @@ -418,7 +424,7 @@ func EnableComponent(ctx context.Context, instances []spec.Instance, noAgentHost case spec.ComponentNodeExporter, spec.ComponentBlackboxExporter: if noAgentHosts.Exist(ins.GetHost()) { - log.Debugf("Ignored enabling/disabling %s for %s:%d", name, ins.GetHost(), ins.GetPort()) + logger.Debugf("Ignored enabling/disabling %s for %s:%d", name, ins.GetHost(), ins.GetPort()) continue } } @@ -445,8 +451,9 @@ func StartComponent(ctx context.Context, instances []spec.Instance, noAgentHosts return nil } + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) name := instances[0].ComponentName() - log.Infof("Starting component %s", name) + logger.Infof("Starting component %s", name) // start instances in serial for Raft related components // eg: PD has more strict restrictions on the capacity expansion process, @@ -469,7 +476,7 @@ func StartComponent(ctx context.Context, instances []spec.Instance, noAgentHosts case spec.ComponentNodeExporter, spec.ComponentBlackboxExporter: if noAgentHosts.Exist(ins.GetHost()) { - log.Debugf("Ignored starting %s for %s:%d", name, ins.GetHost(), ins.GetPort()) + logger.Debugf("Ignored starting %s for %s:%d", name, ins.GetHost(), ins.GetPort()) continue } } @@ -503,13 +510,14 @@ func serialStartInstances(ctx context.Context, instances []spec.Instance, option func stopInstance(ctx context.Context, ins spec.Instance, timeout uint64) error { e := ctxt.GetInner(ctx).Get(ins.GetHost()) - log.Infof("\tStopping instance %s", ins.GetHost()) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + logger.Infof("\tStopping instance %s", ins.GetHost()) if err := systemctl(ctx, e, ins.ServiceName(), "stop", timeout); err != nil { return toFailedActionError(err, "stop", ins.GetHost(), ins.ServiceName(), ins.LogDir()) } - log.Infof("\tStop %s %s success", ins.ComponentName(), ins.ID()) + logger.Infof("\tStop %s %s success", ins.ComponentName(), ins.ID()) return nil } @@ -520,8 +528,9 @@ func StopComponent(ctx context.Context, instances []spec.Instance, noAgentHosts return nil } + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) name := instances[0].ComponentName() - log.Infof("Stopping component %s", name) + logger.Infof("Stopping component %s", name) errg, _ := errgroup.WithContext(ctx) @@ -531,7 +540,7 @@ func StopComponent(ctx context.Context, instances []spec.Instance, noAgentHosts case spec.ComponentNodeExporter, spec.ComponentBlackboxExporter: if noAgentHosts.Exist(ins.GetHost()) { - log.Debugf("Ignored stopping %s for %s:%d", name, ins.GetHost(), ins.GetPort()) + logger.Debugf("Ignored stopping %s for %s:%d", name, ins.GetHost(), ins.GetPort()) continue } } @@ -555,13 +564,14 @@ func StopComponent(ctx context.Context, instances []spec.Instance, noAgentHosts // PrintClusterStatus print cluster status into the io.Writer. func PrintClusterStatus(ctx context.Context, cluster *spec.Specification) (health bool) { health = true + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) for _, com := range cluster.ComponentsByStartOrder() { if len(com.Instances()) == 0 { continue } - log.Infof("Checking service state of %s", com.Name()) + logger.Infof("Checking service state of %s", com.Name()) errg, _ := errgroup.WithContext(ctx) for _, ins := range com.Instances() { ins := ins @@ -575,9 +585,9 @@ func PrintClusterStatus(ctx context.Context, cluster *spec.Specification) (healt active, err := GetServiceStatus(nctx, e, ins.ServiceName()) if err != nil { health = false - log.Errorf("\t%s\t%v", ins.GetHost(), err) + logger.Errorf("\t%s\t%v", ins.GetHost(), err) } else { - log.Infof("\t%s\t%s", ins.GetHost(), active) + logger.Infof("\t%s\t%s", ins.GetHost(), active) } return nil }) diff --git a/pkg/cluster/operation/check.go b/pkg/cluster/operation/check.go index 56a797a777..032f5b20e4 100644 --- a/pkg/cluster/operation/check.go +++ b/pkg/cluster/operation/check.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/module" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + "go.uber.org/zap" ) // CheckOptions control the list of checks to be performed @@ -197,7 +197,7 @@ func checkNTP(ntpInfo *insight.TimeStat) *CheckResult { } if ntpInfo.Status == "none" { - log.Infof("The NTPd daemon may be not installed, skip.") + zap.L().Info("The NTPd daemon may be not installed, skip.") return result } diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index 5481ca42c1..35e8f74386 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -31,7 +31,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/module" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/set" ) @@ -93,6 +93,7 @@ func StopAndDestroyInstance(ctx context.Context, cluster spec.Topology, instance ignoreErr := options.Force compName := instance.ComponentName() noAgentHosts := set.NewStringSet() + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) cluster.IterInstance(func(inst spec.Instance) { if inst.IgnoreMonitorAgent() { @@ -105,13 +106,13 @@ func StopAndDestroyInstance(ctx context.Context, cluster spec.Topology, instance if !ignoreErr { return errors.Annotatef(err, "failed to stop %s", compName) } - log.Warnf("failed to stop %s: %v", compName, err) + logger.Warnf("failed to stop %s: %v", compName, err) } if err := DestroyComponent(ctx, []spec.Instance{instance}, cluster, options); err != nil { if !ignoreErr { return errors.Annotatef(err, "failed to destroy %s", compName) } - log.Warnf("failed to destroy %s: %v", compName, err) + logger.Warnf("failed to destroy %s: %v", compName, err) } if destroyNode { @@ -123,13 +124,13 @@ func StopAndDestroyInstance(ctx context.Context, cluster spec.Topology, instance if !ignoreErr { return errors.Annotatef(err, "failed to stop monitor") } - log.Warnf("failed to stop %s: %v", "monitor", err) + logger.Warnf("failed to stop %s: %v", "monitor", err) } if err := DestroyMonitored(ctx, instance, monitoredOptions, options.OptTimeout); err != nil { if !ignoreErr { return errors.Annotatef(err, "failed to destroy monitor") } - log.Warnf("failed to destroy %s: %v", "monitor", err) + logger.Warnf("failed to destroy %s: %v", "monitor", err) } } @@ -137,7 +138,7 @@ func StopAndDestroyInstance(ctx context.Context, cluster spec.Topology, instance if !ignoreErr { return errors.Annotatef(err, "failed to delete public key") } - log.Warnf("failed to delete public key") + logger.Warnf("failed to delete public key") } } return nil @@ -150,14 +151,15 @@ func DeleteGlobalDirs(ctx context.Context, host string, options *spec.GlobalOpti } e := ctxt.GetInner(ctx).Get(host) - log.Infof("Clean global directories %s", host) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + logger.Infof("Clean global directories %s", host) for _, dir := range []string{options.LogDir, options.DeployDir, options.DataDir} { if dir == "" { continue } dir = spec.Abs(options.User, dir) - log.Infof("\tClean directory %s on instance %s", dir, host) + logger.Infof("\tClean directory %s on instance %s", dir, host) c := module.ShellModuleConfig{ Command: fmt.Sprintf("rmdir %s > /dev/null 2>&1 || true", dir), @@ -171,7 +173,7 @@ func DeleteGlobalDirs(ctx context.Context, host string, options *spec.GlobalOpti fmt.Println(string(stdout)) } if len(stderr) > 0 { - log.Errorf(string(stderr)) + logger.Errorf(string(stderr)) } if err != nil { @@ -179,14 +181,15 @@ func DeleteGlobalDirs(ctx context.Context, host string, options *spec.GlobalOpti } } - log.Infof("Clean global directories %s success", host) + logger.Infof("Clean global directories %s success", host) return nil } // DeletePublicKey deletes the SSH public key from host func DeletePublicKey(ctx context.Context, host string) error { e := ctxt.GetInner(ctx).Get(host) - log.Infof("Delete public key %s", host) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + logger.Infof("Delete public key %s", host) _, pubKeyPath := ctxt.GetInner(ctx).GetSSHKeySet() publicKey, err := os.ReadFile(pubKeyPath) if err != nil { @@ -209,23 +212,24 @@ func DeletePublicKey(ctx context.Context, host string) error { fmt.Println(string(stdout)) } if len(stderr) > 0 { - log.Errorf(string(stderr)) + logger.Errorf(string(stderr)) } if err != nil { return errors.Annotatef(err, "failed to delete pulblic key on: %s", host) } - log.Infof("Delete public key %s success", host) + logger.Infof("Delete public key %s success", host) return nil } // DestroyMonitored destroy the monitored service. func DestroyMonitored(ctx context.Context, inst spec.Instance, options *spec.MonitoredOptions, timeout uint64) error { e := ctxt.GetInner(ctx).Get(inst.GetHost()) - log.Infof("Destroying monitored %s", inst.GetHost()) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) - log.Infof("\tDestroying instance %s", inst.GetHost()) + logger.Infof("Destroying monitored %s", inst.GetHost()) + logger.Infof("\tDestroying instance %s", inst.GetHost()) // Stop by systemd. delPaths := make([]string, 0) @@ -238,7 +242,7 @@ func DestroyMonitored(ctx context.Context, inst spec.Instance, options *spec.Mon if !inst.IsImported() { delPaths = append(delPaths, options.DeployDir) } else { - log.Warnf("Monitored deploy dir %s not deleted for TiDB-Ansible imported instance %s.", + logger.Warnf("Monitored deploy dir %s not deleted for TiDB-Ansible imported instance %s.", options.DeployDir, inst.InstanceName()) } @@ -258,7 +262,7 @@ func DestroyMonitored(ctx context.Context, inst spec.Instance, options *spec.Mon fmt.Println(string(stdout)) } if len(stderr) > 0 { - log.Errorf(string(stderr)) + logger.Errorf(string(stderr)) } if err != nil { @@ -267,26 +271,27 @@ func DestroyMonitored(ctx context.Context, inst spec.Instance, options *spec.Mon if err := spec.PortStopped(ctx, e, options.NodeExporterPort, timeout); err != nil { str := fmt.Sprintf("%s failed to destroy node exportoer: %s", inst.GetHost(), err) - log.Errorf(str) + logger.Errorf(str) return errors.Annotatef(err, str) } if err := spec.PortStopped(ctx, e, options.BlackboxExporterPort, timeout); err != nil { str := fmt.Sprintf("%s failed to destroy blackbox exportoer: %s", inst.GetHost(), err) - log.Errorf(str) + logger.Errorf(str) return errors.Annotatef(err, str) } - log.Infof("Destroy monitored on %s success", inst.GetHost()) + logger.Infof("Destroy monitored on %s success", inst.GetHost()) return nil } // CleanupComponent cleanup the instances func CleanupComponent(ctx context.Context, delFileMaps map[string]set.StringSet) error { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) for host, delFiles := range delFileMaps { e := ctxt.GetInner(ctx).Get(host) - log.Infof("Cleanup instance %s", host) - log.Debugf("Deleting paths on %s: %s", host, strings.Join(delFiles.Slice(), " ")) + logger.Infof("Cleanup instance %s", host) + logger.Debugf("Deleting paths on %s: %s", host, strings.Join(delFiles.Slice(), " ")) c := module.ShellModuleConfig{ Command: fmt.Sprintf("rm -rf %s;", strings.Join(delFiles.Slice(), " ")), Sudo: true, // the .service files are in a directory owned by root @@ -300,14 +305,14 @@ func CleanupComponent(ctx context.Context, delFileMaps map[string]set.StringSet) fmt.Println(string(stdout)) } if len(stderr) > 0 { - log.Errorf(string(stderr)) + logger.Errorf(string(stderr)) } if err != nil { return errors.Annotatef(err, "failed to cleanup: %s", host) } - log.Infof("Cleanup %s success", host) + logger.Infof("Cleanup %s success", host) } return nil @@ -319,8 +324,9 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T return nil } + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) name := instances[0].ComponentName() - log.Infof("Destroying component %s", name) + logger.Infof("Destroying component %s", name) retainDataRoles := set.NewStringSet(options.RetainDataRoles...) retainDataNodes := set.NewStringSet(options.RetainDataNodes...) @@ -331,7 +337,7 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) e := ctxt.GetInner(ctx).Get(ins.GetHost()) - log.Infof("Destroying instance %s", ins.GetHost()) + logger.Infof("Destroying instance %s", ins.GetHost()) var dataDirs []string if len(ins.DataDir()) > 0 { @@ -363,7 +369,7 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T if !strings.HasPrefix(logDir, ins.DeployDir()) && cls.CountDir(ins.GetHost(), logDir) == 1 { delPaths.Insert(logDir) } - log.Warnf("Deploy dir %s not deleted for TiDB-Ansible imported instance %s.", + logger.Warnf("Deploy dir %s not deleted for TiDB-Ansible imported instance %s.", ins.DeployDir(), ins.InstanceName()) } else { if keepDeployDir { @@ -402,7 +408,7 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T if svc := ins.ServiceName(); svc != "" { delPaths.Insert(fmt.Sprintf("/etc/systemd/system/%s", svc)) } - log.Debugf("Deleting paths on %s: %s", ins.GetHost(), strings.Join(delPaths.Slice(), " ")) + logger.Debugf("Deleting paths on %s: %s", ins.GetHost(), strings.Join(delPaths.Slice(), " ")) c := module.ShellModuleConfig{ Command: fmt.Sprintf("rm -rf %s;", strings.Join(delPaths.Slice(), " ")), Sudo: true, // the .service files are in a directory owned by root @@ -416,15 +422,15 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T fmt.Println(string(stdout)) } if len(stderr) > 0 { - log.Errorf(string(stderr)) + logger.Errorf(string(stderr)) } if err != nil { return errors.Annotatef(err, "failed to destroy: %s", ins.GetHost()) } - log.Infof("Destroy %s success", ins.GetHost()) - log.Infof("- Destroy %s paths: %v", ins.ComponentName(), delPaths.Slice()) + logger.Infof("Destroy %s success", ins.GetHost()) + logger.Infof("- Destroy %s paths: %v", ins.ComponentName(), delPaths.Slice()) } return nil @@ -451,6 +457,7 @@ func DestroyClusterTombstone( options Options, tlsCfg *tls.Config, ) (nodes []string, err error) { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) instCount := map[string]int{} for _, component := range cluster.ComponentsByStartOrder() { for _, instance := range component.Instances() { @@ -463,12 +470,12 @@ func DestroyClusterTombstone( if forcePDEndpoints != "" { pdEndpoints = strings.Split(forcePDEndpoints, ",") - log.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) + logger.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) } else { pdEndpoints = cluster.GetPDList() } - var pdClient = api.NewPDClient(pdEndpoints, 10*time.Second, tlsCfg) + var pdClient = api.NewPDClient(ctx, pdEndpoints, 10*time.Second, tlsCfg) tcpProxy := proxy.GetTCPProxy() if tcpProxy != nil { @@ -498,7 +505,7 @@ func DestroyClusterTombstone( err := StopAndDestroyInstance(ctx, cluster, instance, options, instCount[instance.GetHost()] == 0) if err != nil { if options.Force { - log.Warnf("failed to stop and destroy instance %s (%s), ignored as --force is set, you may need to manually cleanup the files", + logger.Warnf("failed to stop and destroy instance %s (%s), ignored as --force is set, you may need to manually cleanup the files", instance, err) } else { return err diff --git a/pkg/cluster/operation/scale_in.go b/pkg/cluster/operation/scale_in.go index 108ab52394..2999e98da2 100644 --- a/pkg/cluster/operation/scale_in.go +++ b/pkg/cluster/operation/scale_in.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -89,6 +89,7 @@ func ScaleInCluster( options Options, tlsCfg *tls.Config, ) error { + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) // instances by uuid instances := map[string]spec.Instance{} instCount := map[string]int{} @@ -118,7 +119,7 @@ func ScaleInCluster( } if skipTopoCheck { - log.Warnf("%s is set, topology checks ignored, the cluster might be broken after the operations!", EnvNameSkipScaleInTopoCheck) + logger.Warnf("%s is set, topology checks ignored, the cluster might be broken after the operations!", EnvNameSkipScaleInTopoCheck) if ok, input := tui.PromptForConfirmYes("Are you sure to continue? [y/N]"); !ok { return errors.Errorf("user aborted with '%s'", input) } @@ -157,7 +158,7 @@ func ScaleInCluster( if forcePDEndpoints != "" { pdEndpoints = strings.Split(forcePDEndpoints, ",") - log.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) + logger.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) } else { for _, instance := range (&spec.PDComponent{Topology: cluster}).Instances() { if !deletedNodes.Exist(instance.ID()) { @@ -171,7 +172,7 @@ func ScaleInCluster( return errors.New("cannot find available PD instance") } - pdClient := api.NewPDClient(pdEndpoints, 10*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, pdEndpoints, 10*time.Second, tlsCfg) tcpProxy := proxy.GetTCPProxy() if tcpProxy != nil { @@ -194,13 +195,13 @@ func ScaleInCluster( if compName != spec.ComponentPump && compName != spec.ComponentDrainer { if err := deleteMember(ctx, component, instance, pdClient, binlogClient, options.APITimeout); err != nil { - log.Warnf("failed to delete %s: %v", compName, err) + logger.Warnf("failed to delete %s: %v", compName, err) } } instCount[instance.GetHost()]-- if err := StopAndDestroyInstance(ctx, cluster, instance, options, instCount[instance.GetHost()] == 0); err != nil { - log.Warnf("failed to stop/destroy %s: %v", compName, err) + logger.Warnf("failed to stop/destroy %s: %v", compName, err) } // directly update pump&drainer 's state as offline in etcd. @@ -208,11 +209,11 @@ func ScaleInCluster( id := instance.ID() if compName == spec.ComponentPump { if err := binlogClient.UpdatePumpState(ctx, id, "offline"); err != nil { - log.Warnf("failed to update %s state as offline: %v", compName, err) + logger.Warnf("failed to update %s state as offline: %v", compName, err) } } else if compName == spec.ComponentDrainer { if err := binlogClient.UpdateDrainerState(ctx, id, "offline"); err != nil { - log.Warnf("failed to update %s state as offline: %v", compName, err) + logger.Warnf("failed to update %s state as offline: %v", compName, err) } } } @@ -254,7 +255,7 @@ func ScaleInCluster( maxReplicas := config.MaxReplicas if len(tikvInstances) < maxReplicas { - log.Warnf(fmt.Sprintf("TiKV instance number %d will be less than max-replicas setting after scale-in. TiFlash won't be able to receive data from leader before TiKV instance number reach %d", len(tikvInstances), maxReplicas)) + logger.Warnf(fmt.Sprintf("TiKV instance number %d will be less than max-replicas setting after scale-in. TiFlash won't be able to receive data from leader before TiKV instance number reach %d", len(tikvInstances), maxReplicas)) } } @@ -276,7 +277,7 @@ func ScaleInCluster( return err } } else { - log.Warnf(color.YellowString("The component `%s` will become tombstone, maybe exists in several minutes or hours, after that you can use the prune command to clean it", + logger.Warnf(color.YellowString("The component `%s` will become tombstone, maybe exists in several minutes or hours, after that you can use the prune command to clean it", component.Name())) } } diff --git a/pkg/cluster/operation/upgrade.go b/pkg/cluster/operation/upgrade.go index 1693c4bdba..cf6cf20d6e 100644 --- a/pkg/cluster/operation/upgrade.go +++ b/pkg/cluster/operation/upgrade.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tiup/pkg/checkpoint" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "go.uber.org/zap" ) @@ -49,6 +49,7 @@ func Upgrade( nodeFilter := set.NewStringSet(options.Nodes...) components := topo.ComponentsByUpdateOrder() components = FilterComponent(components, roleFilter) + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) for _, component := range components { instances := FilterInstance(component.Instances(), nodeFilter) @@ -56,7 +57,7 @@ func Upgrade( continue } - log.Infof("Upgrading component %s", component.Name()) + logger.Infof("Upgrading component %s", component.Name()) // perform pre-upgrade actions of component var origLeaderScheduleLimit int @@ -70,21 +71,21 @@ func Upgrade( case spec.ComponentTiKV: if forcePDEndpoints != "" { pdEndpoints = strings.Split(forcePDEndpoints, ",") - log.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) + logger.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) } else { pdEndpoints = topo.(*spec.Specification).GetPDList() } - pdClient := api.NewPDClient(pdEndpoints, 10*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, pdEndpoints, 10*time.Second, tlsCfg) origLeaderScheduleLimit, origRegionScheduleLimit, err = increaseScheduleLimit(ctx, pdClient) if err != nil { // the config modifing error should be able to be safely ignored, as it will // be processed with current settings anyway. - log.Warnf("failed increasing schedule limit: %s, ignore", err) + logger.Warnf("failed increasing schedule limit: %s, ignore", err) } else { defer func() { upgErr := decreaseScheduleLimit(pdClient, origLeaderScheduleLimit, origRegionScheduleLimit) if upgErr != nil { - log.Warnf( + logger.Warnf( "failed decreasing schedule limit (original values should be: %s, %s), please check if their current values are reasonable: %s", fmt.Sprintf("leader-schedule-limit=%d", origLeaderScheduleLimit), fmt.Sprintf("region-schedule-limit=%d", origRegionScheduleLimit), @@ -104,13 +105,13 @@ func Upgrade( switch component.Name() { case spec.ComponentPD: // defer PD leader to be upgraded after others - isLeader, err := instance.(*spec.PDInstance).IsLeader(topo, int(options.APITimeout), tlsCfg) + isLeader, err := instance.(*spec.PDInstance).IsLeader(ctx, topo, int(options.APITimeout), tlsCfg) if err != nil { return err } if isLeader { deferInstances = append(deferInstances, instance) - log.Debugf("Defferred upgrading of PD leader %s", instance.ID()) + logger.Debugf("Defferred upgrading of PD leader %s", instance.ID()) continue } default: @@ -124,7 +125,7 @@ func Upgrade( // process defferred instances for _, instance := range deferInstances { - log.Debugf("Upgrading defferred instance %s...", instance.ID()) + logger.Debugf("Upgrading defferred instance %s...", instance.ID()) if err := upgradeInstance(ctx, topo, instance, options, tlsCfg); err != nil { return err } @@ -153,7 +154,7 @@ func upgradeInstance(ctx context.Context, topo spec.Topology, instance spec.Inst } if isRollingInstance { - err := rollingInstance.PreRestart(topo, int(options.APITimeout), tlsCfg) + err := rollingInstance.PreRestart(ctx, topo, int(options.APITimeout), tlsCfg) if err != nil && !options.Force { return err } @@ -164,7 +165,7 @@ func upgradeInstance(ctx context.Context, topo spec.Topology, instance spec.Inst } if isRollingInstance { - err := rollingInstance.PostRestart(topo, tlsCfg) + err := rollingInstance.PostRestart(ctx, topo, tlsCfg) if err != nil && !options.Force { return err } diff --git a/pkg/cluster/spec/alertmanager.go b/pkg/cluster/spec/alertmanager.go index 47b52e18c7..be3c04adc1 100644 --- a/pkg/cluster/spec/alertmanager.go +++ b/pkg/cluster/spec/alertmanager.go @@ -106,10 +106,10 @@ func (c *AlertManagerComponent) Instances() []Instance { s.DeployDir, s.DataDir, }, - StatusFn: func(_ *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, _ *tls.Config, _ ...string) string { return statusByHost(s.Host, s.WebPort, "/-/ready", nil) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.WebPort, tlsCfg) }, }, diff --git a/pkg/cluster/spec/cdc.go b/pkg/cluster/spec/cdc.go index fc3dd75c1c..7a53b11921 100644 --- a/pkg/cluster/spec/cdc.go +++ b/pkg/cluster/spec/cdc.go @@ -104,10 +104,10 @@ func (c *CDCComponent) Instances() []Instance { Dirs: []string{ s.DeployDir, }, - StatusFn: func(tlsCfg *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, tlsCfg *tls.Config, _ ...string) string { return statusByHost(s.Host, s.Port, "/status", tlsCfg) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.Port, tlsCfg) }, }, c.Topology} diff --git a/pkg/cluster/spec/drainer.go b/pkg/cluster/spec/drainer.go index 542b0c62d5..160e88db49 100644 --- a/pkg/cluster/spec/drainer.go +++ b/pkg/cluster/spec/drainer.go @@ -104,10 +104,10 @@ func (c *DrainerComponent) Instances() []Instance { s.DeployDir, s.DataDir, }, - StatusFn: func(tlsCfg *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, tlsCfg *tls.Config, _ ...string) string { return statusByHost(s.Host, s.Port, "/status", tlsCfg) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.Port, tlsCfg) }, }, c.Topology}) diff --git a/pkg/cluster/spec/grafana.go b/pkg/cluster/spec/grafana.go index 748d470590..285ddc44a7 100644 --- a/pkg/cluster/spec/grafana.go +++ b/pkg/cluster/spec/grafana.go @@ -110,10 +110,10 @@ func (c *GrafanaComponent) Instances() []Instance { Dirs: []string{ s.DeployDir, }, - StatusFn: func(_ *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, _ *tls.Config, _ ...string) string { return statusByHost(s.Host, s.Port, "", nil) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.Port, tlsCfg) }, }, diff --git a/pkg/cluster/spec/grafana_test.go b/pkg/cluster/spec/grafana_test.go index c5867d013c..437c54a58a 100644 --- a/pkg/cluster/spec/grafana_test.go +++ b/pkg/cluster/spec/grafana_test.go @@ -24,12 +24,13 @@ import ( "github.com/google/uuid" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/executor" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/stretchr/testify/assert" ) func TestLocalDashboards(t *testing.T) { - ctx := ctxt.New(context.Background(), 0) + ctx := ctxt.New(context.Background(), 0, logprinter.NewLogger("")) deployDir, err := os.MkdirTemp("", "tiup-*") assert.Nil(t, err) diff --git a/pkg/cluster/spec/instance.go b/pkg/cluster/spec/instance.go index c06a0d6f98..0bcd46b387 100644 --- a/pkg/cluster/spec/instance.go +++ b/pkg/cluster/spec/instance.go @@ -73,8 +73,8 @@ type Component interface { // RollingUpdateInstance represent a instance need to transfer state when restart. // e.g transfer leader. type RollingUpdateInstance interface { - PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error - PostRestart(topo Topology, tlsCfg *tls.Config) error + PreRestart(ctx context.Context, topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error + PostRestart(ctx context.Context, topo Topology, tlsCfg *tls.Config) error } // Instance represents the instance. @@ -94,8 +94,8 @@ type Instance interface { DeployDir() string UsedPorts() []int UsedDirs() []string - Status(tlsCfg *tls.Config, pdList ...string) string - Uptime(tlsCfg *tls.Config) time.Duration + Status(ctx context.Context, tlsCfg *tls.Config, pdList ...string) string + Uptime(ctx context.Context, tlsCfg *tls.Config) time.Duration DataDir() string LogDir() string OS() string // only linux supported now @@ -139,8 +139,8 @@ type BaseInstance struct { Ports []int Dirs []string - StatusFn func(tlsCfg *tls.Config, pdHosts ...string) string - UptimeFn func(tlsCfg *tls.Config) time.Duration + StatusFn func(ctx context.Context, tlsCfg *tls.Config, pdHosts ...string) string + UptimeFn func(ctx context.Context, tlsCfg *tls.Config) time.Duration } // Ready implements Instance interface @@ -464,11 +464,11 @@ func (i *BaseInstance) UsedDirs() []string { } // Status implements Instance interface -func (i *BaseInstance) Status(tlsCfg *tls.Config, pdList ...string) string { - return i.StatusFn(tlsCfg, pdList...) +func (i *BaseInstance) Status(ctx context.Context, tlsCfg *tls.Config, pdList ...string) string { + return i.StatusFn(ctx, tlsCfg, pdList...) } // Uptime implements Instance interface -func (i *BaseInstance) Uptime(tlsCfg *tls.Config) time.Duration { - return i.UptimeFn(tlsCfg) +func (i *BaseInstance) Uptime(ctx context.Context, tlsCfg *tls.Config) time.Duration { + return i.UptimeFn(ctx, tlsCfg) } diff --git a/pkg/cluster/spec/monitoring.go b/pkg/cluster/spec/monitoring.go index f7063ae0e0..7deb37562f 100644 --- a/pkg/cluster/spec/monitoring.go +++ b/pkg/cluster/spec/monitoring.go @@ -17,6 +17,7 @@ import ( "context" "crypto/tls" "fmt" + "os" "path" "path/filepath" "reflect" @@ -29,6 +30,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/set" + "gopkg.in/yaml.v3" ) // PrometheusSpec represents the Prometheus Server topology specification in topology.yaml @@ -51,6 +53,7 @@ type PrometheusSpec struct { Arch string `yaml:"arch,omitempty"` OS string `yaml:"os,omitempty"` RuleDir string `yaml:"rule_dir,omitempty" validate:"rule_dir:editable"` + AdditionalScrapeConf map[string]interface{} `yaml:"additional_scrape_conf,omitempty" validate:"additional_scrape_conf:ignore"` } // Remote prometheus remote config @@ -123,10 +126,10 @@ func (c *MonitorComponent) Instances() []Instance { s.DeployDir, s.DataDir, }, - StatusFn: func(_ *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, _ *tls.Config, _ ...string) string { return statusByHost(s.Host, s.Port, "/-/ready", nil) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.Port, tlsCfg) }, }, c.Topology} @@ -268,6 +271,7 @@ func (i *MonitorInstance) InitConfig( for i := 0; i < servers.Len(); i++ { master := reflect.Indirect(servers.Index(i)) host, port := master.FieldByName("Host").String(), master.FieldByName("Port").Int() + uniqueHosts.Insert(host) cfig.AddDMMaster(host, uint64(port)) } } @@ -276,6 +280,7 @@ func (i *MonitorInstance) InitConfig( for i := 0; i < servers.Len(); i++ { worker := reflect.Indirect(servers.Index(i)) host, port := worker.FieldByName("Host").String(), worker.FieldByName("Port").Int() + uniqueHosts.Insert(host) cfig.AddDMWorker(host, uint64(port)) } } @@ -342,6 +347,12 @@ func (i *MonitorInstance) InitConfig( if err := cfig.ConfigToFile(fp); err != nil { return err } + if spec.AdditionalScrapeConf != nil { + err = mergeAdditionalScrapeConf(fp, spec.AdditionalScrapeConf) + if err != nil { + return err + } + } dst = filepath.Join(paths.Deploy, "conf", "prometheus.yml") if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err @@ -439,3 +450,26 @@ func (i *MonitorInstance) ScaleConfig( i.topo = topo return i.InitConfig(ctx, e, clusterName, clusterVersion, deployUser, paths) } + +func mergeAdditionalScrapeConf(source string, addition map[string]interface{}) error { + var result map[string]interface{} + bytes, err := os.ReadFile(source) + if err != nil { + return err + } + err = yaml.Unmarshal(bytes, &result) + if err != nil { + return err + } + + for _, job := range result["scrape_configs"].([]interface{}) { + for k, v := range addition { + job.(map[string]interface{})[k] = v + } + } + bytes, err = yaml.Marshal(result) + if err != nil { + return err + } + return os.WriteFile(source, bytes, 0644) +} diff --git a/pkg/cluster/spec/monitoring_test.go b/pkg/cluster/spec/monitoring_test.go index ae3d0a07c1..a0a85dfb2a 100644 --- a/pkg/cluster/spec/monitoring_test.go +++ b/pkg/cluster/spec/monitoring_test.go @@ -15,6 +15,7 @@ package spec import ( "context" + "fmt" "os" "os/user" "path" @@ -25,6 +26,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/meta" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestLocalRuleDirs(t *testing.T) { @@ -128,3 +130,95 @@ groups: assert.NoFileExists(t, path.Join(deployDir, "conf", f.Name())) } } + +func TestMergeAdditionalScrapeConf(t *testing.T) { + file, err := os.CreateTemp("", "tiup-cluster-spec-test") + if err != nil { + panic(fmt.Sprintf("create temp file: %s", err)) + } + defer os.Remove(file.Name()) + + _, err = file.WriteString(`--- +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + evaluation_interval: 15s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + external_labels: + cluster: 'test' + monitor: "prometheus" + +scrape_configs: + - job_name: "tidb" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: + - '192.168.122.215:10080' + - job_name: "tikv" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: + - '192.168.122.25:20180'`) + assert.Nil(t, err) + + expected := `global: + evaluation_interval: 15s + external_labels: + cluster: test + monitor: prometheus + scrape_interval: 15s +scrape_configs: + - honor_labels: true + job_name: tidb + metric_relabel_configs: + - action: drop + regex: tikv_thread_nonvoluntary_context_switches|tikv_thread_voluntary_context_switches|tikv_threads_io_bytes_total + separator: ; + source_labels: + - __name__ + - action: drop + regex: tikv_thread_cpu_seconds_total;(tokio|rocksdb).+ + separator: ; + source_labels: + - __name__ + - name + static_configs: + - targets: + - 192.168.122.215:10080 + - honor_labels: true + job_name: tikv + metric_relabel_configs: + - action: drop + regex: tikv_thread_nonvoluntary_context_switches|tikv_thread_voluntary_context_switches|tikv_threads_io_bytes_total + separator: ; + source_labels: + - __name__ + - action: drop + regex: tikv_thread_cpu_seconds_total;(tokio|rocksdb).+ + separator: ; + source_labels: + - __name__ + - name + static_configs: + - targets: + - 192.168.122.25:20180 +` + + var addition map[string]interface{} + err = yaml.Unmarshal([]byte(`metric_relabel_configs: + - source_labels: [__name__] + separator: ; + regex: tikv_thread_nonvoluntary_context_switches|tikv_thread_voluntary_context_switches|tikv_threads_io_bytes_total + action: drop + - source_labels: [__name__,name] + separator: ; + regex: tikv_thread_cpu_seconds_total;(tokio|rocksdb).+ + action: drop`), &addition) + assert.Nil(t, err) + + err = mergeAdditionalScrapeConf(file.Name(), addition) + assert.Nil(t, err) + result, err := os.ReadFile(file.Name()) + assert.Nil(t, err) + + assert.Equal(t, expected, string(result)) +} diff --git a/pkg/cluster/spec/pd.go b/pkg/cluster/spec/pd.go index e913dbb093..fd47a02769 100644 --- a/pkg/cluster/spec/pd.go +++ b/pkg/cluster/spec/pd.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/template/scripts" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/utils" ) @@ -55,9 +54,9 @@ type PDSpec struct { } // Status queries current status of the instance -func (s *PDSpec) Status(tlsCfg *tls.Config, _ ...string) string { +func (s *PDSpec) Status(ctx context.Context, tlsCfg *tls.Config, _ ...string) string { addr := fmt.Sprintf("%s:%d", s.Host, s.ClientPort) - pc := api.NewPDClient([]string{addr}, statusQueryTimeout, tlsCfg) + pc := api.NewPDClient(ctx, []string{addr}, statusQueryTimeout, tlsCfg) // check health err := pc.CheckHealth() @@ -139,7 +138,7 @@ func (c *PDComponent) Instances() []Instance { s.DataDir, }, StatusFn: s.Status, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.ClientPort, tlsCfg) }, }, @@ -310,7 +309,6 @@ func (i *PDInstance) ScaleConfig( cfg := scripts.NewPDScaleScript(cfg0) fp := filepath.Join(paths.Cache, fmt.Sprintf("run_pd_%s_%d.sh", i.GetHost(), i.GetPort())) - log.Infof("script path: %s", fp) if err := cfg.ConfigToFile(fp); err != nil { return err } @@ -328,12 +326,12 @@ func (i *PDInstance) ScaleConfig( var _ RollingUpdateInstance = &PDInstance{} // IsLeader checks if the instance is PD leader -func (i *PDInstance) IsLeader(topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) (bool, error) { +func (i *PDInstance) IsLeader(ctx context.Context, topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) (bool, error) { tidbTopo, ok := topo.(*Specification) if !ok { panic("topo should be type of tidb topology") } - pdClient := api.NewPDClient(tidbTopo.GetPDList(), time.Second*5, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), time.Second*5, tlsCfg) return i.isLeader(pdClient) } @@ -348,7 +346,7 @@ func (i *PDInstance) isLeader(pdClient *api.PDClient) (bool, error) { } // PreRestart implements RollingUpdateInstance interface. -func (i *PDInstance) PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error { +func (i *PDInstance) PreRestart(ctx context.Context, topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error { timeoutOpt := &utils.RetryOption{ Timeout: time.Second * time.Duration(apiTimeoutSeconds), Delay: time.Second * 2, @@ -358,7 +356,7 @@ func (i *PDInstance) PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg *tl if !ok { panic("topo should be type of tidb topology") } - pdClient := api.NewPDClient(tidbTopo.GetPDList(), time.Second*5, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), time.Second*5, tlsCfg) isLeader, err := i.isLeader(pdClient) if err != nil { @@ -374,7 +372,7 @@ func (i *PDInstance) PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg *tl } // PostRestart implements RollingUpdateInstance interface. -func (i *PDInstance) PostRestart(topo Topology, tlsCfg *tls.Config) error { +func (i *PDInstance) PostRestart(ctx context.Context, topo Topology, tlsCfg *tls.Config) error { // When restarting the next PD, if the PD has not been fully started and has become the target of // the transfer leader, this may cause the PD service to be unavailable for about 10 seconds. @@ -384,7 +382,7 @@ func (i *PDInstance) PostRestart(topo Topology, tlsCfg *tls.Config) error { Timeout: 120 * time.Second, } currentPDAddrs := []string{fmt.Sprintf("%s:%d", i.Host, i.Port)} - pdClient := api.NewPDClient(currentPDAddrs, 5*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, currentPDAddrs, 5*time.Second, tlsCfg) if err := utils.Retry(pdClient.CheckHealth, timeoutOpt); err != nil { return errors.Annotatef(err, "failed to start PD peer %s", i.GetHost()) diff --git a/pkg/cluster/spec/pump.go b/pkg/cluster/spec/pump.go index bbc461847e..ddea53e97c 100644 --- a/pkg/cluster/spec/pump.go +++ b/pkg/cluster/spec/pump.go @@ -102,10 +102,10 @@ func (c *PumpComponent) Instances() []Instance { s.DeployDir, s.DataDir, }, - StatusFn: func(tlsCfg *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, tlsCfg *tls.Config, _ ...string) string { return statusByHost(s.Host, s.Port, "/status", tlsCfg) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.Port, tlsCfg) }, }, c.Topology}) diff --git a/pkg/cluster/spec/server_config.go b/pkg/cluster/spec/server_config.go index fc468c711a..dc64212857 100644 --- a/pkg/cluster/spec/server_config.go +++ b/pkg/cluster/spec/server_config.go @@ -27,9 +27,9 @@ import ( perrs "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/ctxt" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/utils" + "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -321,7 +321,7 @@ func HandleImportPathMigration(clsName string) error { targetPath := path.Join(dirPath, AnsibleImportedConfigPath) _, err := os.Stat(targetPath) if os.IsNotExist(err) { - log.Warnf("renaming '%s/config' to '%s'", dirPath, targetPath) + zap.L().Warn("renaming config dir", zap.String("orig", dirPath), zap.String("new", targetPath)) if lckErr := utils.Retry(func() error { _, lckErr := os.Stat(path.Join(dirPath, migrateLockName)) @@ -338,7 +338,7 @@ func HandleImportPathMigration(clsName string) error { defer func() { rmErr := os.Remove(path.Join(dirPath, migrateLockName)) if rmErr != nil { - log.Errorf("error unlocking config dir, %s", rmErr) + zap.L().Error("error unlocking config dir", zap.Error(rmErr)) } }() diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index c2739b3eb6..0d264c5f78 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -14,6 +14,7 @@ package spec import ( + "context" "crypto/tls" "fmt" "path/filepath" @@ -27,11 +28,11 @@ import ( "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/template/scripts" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/tui" clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" "golang.org/x/mod/semver" ) @@ -344,7 +345,11 @@ func (s *Specification) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } if s.TiFlashServers[i].DataDir != dataDir { - log.Infof("'tiflash_server:%s.data_dir' is overwritten by its storage configuration. Now the data_dir is %s", s.TiFlashServers[i].Host, dataDir) + zap.L().Info( + "tiflash data dir is overwritten by its storage configuration", + zap.String("host", s.TiFlashServers[i].Host), + zap.String("dir", dataDir), + ) s.TiFlashServers[i].DataDir = dataDir } } @@ -399,8 +404,8 @@ func (s *Specification) AdjustByVersion(clusterVersion string) { } // GetDashboardAddress returns the cluster's dashboard addr -func (s *Specification) GetDashboardAddress(tlsCfg *tls.Config, pdList ...string) (string, error) { - pc := api.NewPDClient(pdList, statusQueryTimeout, tlsCfg) +func (s *Specification) GetDashboardAddress(ctx context.Context, tlsCfg *tls.Config, pdList ...string) (string, error) { + pc := api.NewPDClient(ctx, pdList, statusQueryTimeout, tlsCfg) dashboardAddr, err := pc.GetDashboardAddress() if err != nil { return "", err diff --git a/pkg/cluster/spec/tidb.go b/pkg/cluster/spec/tidb.go index a0b147352c..399777832e 100644 --- a/pkg/cluster/spec/tidb.go +++ b/pkg/cluster/spec/tidb.go @@ -104,10 +104,10 @@ func (c *TiDBComponent) Instances() []Instance { Dirs: []string{ s.DeployDir, }, - StatusFn: func(tlsCfg *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, tlsCfg *tls.Config, _ ...string) string { return statusByHost(s.Host, s.StatusPort, "/status", tlsCfg) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.StatusPort, tlsCfg) }, }, c.Topology}) diff --git a/pkg/cluster/spec/tiflash.go b/pkg/cluster/spec/tiflash.go index c550d996f1..b4851de440 100644 --- a/pkg/cluster/spec/tiflash.go +++ b/pkg/cluster/spec/tiflash.go @@ -66,9 +66,9 @@ type TiFlashSpec struct { } // Status queries current status of the instance -func (s *TiFlashSpec) Status(tlsCfg *tls.Config, pdList ...string) string { +func (s *TiFlashSpec) Status(ctx context.Context, tlsCfg *tls.Config, pdList ...string) string { storeAddr := fmt.Sprintf("%s:%d", s.Host, s.FlashServicePort) - state := checkStoreStatus(storeAddr, tlsCfg, pdList...) + state := checkStoreStatus(ctx, storeAddr, tlsCfg, pdList...) if s.Offline && strings.ToLower(state) == "offline" { state = "Pending Offline" // avoid misleading } @@ -234,7 +234,7 @@ func (c *TiFlashComponent) Instances() []Instance { s.DataDir, }, StatusFn: s.Status, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return 0 }, }, c.Topology}) @@ -784,7 +784,7 @@ func (i *TiFlashInstance) PrepareStart(ctx context.Context, tlsCfg *tls.Config) } endpoints := i.getEndpoints(topo) - pdClient := api.NewPDClient(endpoints, 10*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, endpoints, 10*time.Second, tlsCfg) return pdClient.UpdateReplicateConfig(bytes.NewBuffer(enablePlacementRules)) } diff --git a/pkg/cluster/spec/tikv.go b/pkg/cluster/spec/tikv.go index 54389e1c8b..b4af65e067 100644 --- a/pkg/cluster/spec/tikv.go +++ b/pkg/cluster/spec/tikv.go @@ -30,11 +30,12 @@ import ( "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/template/scripts" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/utils" dto "github.com/prometheus/client_model/go" "github.com/prometheus/prom2json" + "go.uber.org/zap" ) const ( @@ -66,11 +67,11 @@ type TiKVSpec struct { } // checkStoreStatus checks the store status in current cluster -func checkStoreStatus(storeAddr string, tlsCfg *tls.Config, pdList ...string) string { +func checkStoreStatus(ctx context.Context, storeAddr string, tlsCfg *tls.Config, pdList ...string) string { if len(pdList) < 1 { return "N/A" } - pdapi := api.NewPDClient(pdList, statusQueryTimeout, tlsCfg) + pdapi := api.NewPDClient(ctx, pdList, statusQueryTimeout, tlsCfg) store, err := pdapi.GetCurrentStore(storeAddr) if err != nil { if errors.Is(err, api.ErrNoStore) { @@ -83,9 +84,9 @@ func checkStoreStatus(storeAddr string, tlsCfg *tls.Config, pdList ...string) st } // Status queries current status of the instance -func (s *TiKVSpec) Status(tlsCfg *tls.Config, pdList ...string) string { +func (s *TiKVSpec) Status(ctx context.Context, tlsCfg *tls.Config, pdList ...string) string { storeAddr := addr(s) - state := checkStoreStatus(storeAddr, tlsCfg, pdList...) + state := checkStoreStatus(ctx, storeAddr, tlsCfg, pdList...) if s.Offline && strings.ToLower(state) == "offline" { state = "Pending Offline" // avoid misleading } @@ -182,7 +183,7 @@ func (c *TiKVComponent) Instances() []Instance { s.DataDir, }, StatusFn: s.Status, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return UptimeByHost(s.Host, s.StatusPort, tlsCfg) }, }, c.Topology}) @@ -329,7 +330,7 @@ func (i *TiKVInstance) ScaleConfig( var _ RollingUpdateInstance = &TiKVInstance{} // PreRestart implements RollingUpdateInstance interface. -func (i *TiKVInstance) PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error { +func (i *TiKVInstance) PreRestart(ctx context.Context, topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error { timeoutOpt := &utils.RetryOption{ Timeout: time.Second * time.Duration(apiTimeoutSeconds), Delay: time.Second * 2, @@ -344,7 +345,7 @@ func (i *TiKVInstance) PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg * return nil } - pdClient := api.NewPDClient(tidbTopo.GetPDList(), 5*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), 5*time.Second, tlsCfg) // Make sure there's leader of PD. // Although we evict pd leader when restart pd, @@ -356,7 +357,8 @@ func (i *TiKVInstance) PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg * if err := pdClient.EvictStoreLeader(addr(i.InstanceSpec.(*TiKVSpec)), timeoutOpt, genLeaderCounter(tidbTopo, tlsCfg)); err != nil { if utils.IsTimeoutOrMaxRetry(err) { - log.Warnf("Ignore evicting store leader from %s, %v", i.ID(), err) + ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger). + Warnf("Ignore evicting store leader from %s, %v", i.ID(), err) } else { return perrs.Annotatef(err, "failed to evict store leader %s", i.GetHost()) } @@ -365,7 +367,7 @@ func (i *TiKVInstance) PreRestart(topo Topology, apiTimeoutSeconds int, tlsCfg * } // PostRestart implements RollingUpdateInstance interface. -func (i *TiKVInstance) PostRestart(topo Topology, tlsCfg *tls.Config) error { +func (i *TiKVInstance) PostRestart(ctx context.Context, topo Topology, tlsCfg *tls.Config) error { tidbTopo, ok := topo.(*Specification) if !ok { panic("should be type of tidb topology") @@ -375,7 +377,7 @@ func (i *TiKVInstance) PostRestart(topo Topology, tlsCfg *tls.Config) error { return nil } - pdClient := api.NewPDClient(tidbTopo.GetPDList(), 5*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), 5*time.Second, tlsCfg) // remove store leader evict scheduler after restart if err := pdClient.RemoveStoreEvict(addr(i.InstanceSpec.(*TiKVSpec))); err != nil { @@ -425,7 +427,11 @@ func genLeaderCounter(topo *Specification, tlsCfg *tls.Config) func(string) (int } if err := prom2json.FetchMetricFamilies(addr, mfChan, transport); err != nil { - log.Errorf("failed counting leader on %s (status addr %s), %v", id, addr, err) + zap.L().Error("failed counting leader", + zap.String("host", id), + zap.String("status addr", addr), + zap.Error(err), + ) } }() diff --git a/pkg/cluster/spec/tispark.go b/pkg/cluster/spec/tispark.go index b222088888..eb16fa550d 100644 --- a/pkg/cluster/spec/tispark.go +++ b/pkg/cluster/spec/tispark.go @@ -150,10 +150,10 @@ func (c *TiSparkMasterComponent) Instances() []Instance { Dirs: []string{ s.DeployDir, }, - StatusFn: func(tlsCfg *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, tlsCfg *tls.Config, _ ...string) string { return statusByHost(s.Host, s.WebPort, "", tlsCfg) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return 0 }, }, @@ -331,10 +331,10 @@ func (c *TiSparkWorkerComponent) Instances() []Instance { Dirs: []string{ s.DeployDir, }, - StatusFn: func(tlsCfg *tls.Config, _ ...string) string { + StatusFn: func(_ context.Context, tlsCfg *tls.Config, _ ...string) string { return statusByHost(s.Host, s.WebPort, "", tlsCfg) }, - UptimeFn: func(tlsCfg *tls.Config) time.Duration { + UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration { return 0 }, }, diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index dac8e231f5..0a575cfdce 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -18,13 +18,12 @@ import ( "crypto/tls" "fmt" "path/filepath" - "strings" "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/crypto" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/tui" @@ -32,22 +31,13 @@ import ( // Builder is used to build TiUP task type Builder struct { - tasks []Task - DisplayMode log.DisplayMode + tasks []Task + Logger *logprinter.Logger } // NewBuilder returns a *Builder instance -func NewBuilder(mode string) *Builder { - var dp log.DisplayMode - switch strings.ToLower(mode) { - case "json": - dp = log.DisplayModeJSON - case "plain", "text": - dp = log.DisplayModePlain - default: - dp = log.DisplayModeDefault - } - return &Builder{DisplayMode: dp} +func NewBuilder(logger *logprinter.Logger) *Builder { + return &Builder{Logger: logger} } // RootSSH appends a RootSSH task to the current task collection @@ -494,20 +484,20 @@ func (b *Builder) Build() Task { } // Step appends a new StepDisplay task, which will print single line progress for inner tasks. -func (b *Builder) Step(prefix string, inner Task) *Builder { - b.Serial(newStepDisplay(prefix, inner, b.DisplayMode)) +func (b *Builder) Step(prefix string, inner Task, logger *logprinter.Logger) *Builder { + b.Serial(newStepDisplay(prefix, inner, logger)) return b } // ParallelStep appends a new ParallelStepDisplay task, which will print multi line progress in parallel // for inner tasks. Inner tasks must be a StepDisplay task. func (b *Builder) ParallelStep(prefix string, ignoreError bool, tasks ...*StepDisplay) *Builder { - b.tasks = append(b.tasks, newParallelStepDisplay(prefix, ignoreError, tasks...).SetDisplayMode(b.DisplayMode)) + b.tasks = append(b.tasks, newParallelStepDisplay(prefix, ignoreError, tasks...).SetLogger(b.Logger)) return b } // BuildAsStep returns a task that is wrapped by a StepDisplay. The task will print single line progress. func (b *Builder) BuildAsStep(prefix string) *StepDisplay { inner := b.Build() - return newStepDisplay(prefix, inner, b.DisplayMode) + return newStepDisplay(prefix, inner, b.Logger) } diff --git a/pkg/cluster/task/init_config_test.go b/pkg/cluster/task/init_config_test.go index b25450a873..54c9f77676 100644 --- a/pkg/cluster/task/init_config_test.go +++ b/pkg/cluster/task/init_config_test.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/spec" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/check" @@ -73,7 +74,7 @@ func Test(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&initConfigSuite{}) func (s *initConfigSuite) TestCheckConfig(c *check.C) { - ctx := ctxt.New(context.Background(), 0) + ctx := ctxt.New(context.Background(), 0, logprinter.NewLogger("")) defer mock.With("FakeExecutor", &fakeExecutor{})() t := &InitConfig{ diff --git a/pkg/cluster/task/monitored_config.go b/pkg/cluster/task/monitored_config.go index ca733b1149..605de6c7da 100644 --- a/pkg/cluster/task/monitored_config.go +++ b/pkg/cluster/task/monitored_config.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/template/config" "github.com/pingcap/tiup/pkg/cluster/template/scripts" system "github.com/pingcap/tiup/pkg/cluster/template/systemd" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "go.uber.org/zap" ) @@ -121,7 +121,8 @@ func (m *MonitoredConfig) syncMonitoredSystemConfig(ctx context.Context, exec ct fmt.Println(string(outp)) } if len(errp) > 0 { - log.Errorf(string(errp)) + ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger). + Errorf(string(errp)) } return err } diff --git a/pkg/cluster/task/shell.go b/pkg/cluster/task/shell.go index 11d78424b1..e024349132 100644 --- a/pkg/cluster/task/shell.go +++ b/pkg/cluster/task/shell.go @@ -19,7 +19,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/ctxt" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" ) // Shell is used to create directory on the target host @@ -37,7 +37,8 @@ func (m *Shell) Execute(ctx context.Context) error { return ErrNoExecutor } - log.Infof("Run command on %s(sudo:%v): %s", m.host, m.sudo, m.command) + ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger). + Infof("Run command on %s(sudo:%v): %s", m.host, m.sudo, m.command) stdout, stderr, err := exec.Execute(ctx, m.command, m.sudo) outputID := m.host diff --git a/pkg/cluster/task/step.go b/pkg/cluster/task/step.go index 34b223b18e..dc1accfac9 100644 --- a/pkg/cluster/task/step.go +++ b/pkg/cluster/task/step.go @@ -20,7 +20,7 @@ import ( "strings" "github.com/pingcap/tiup/pkg/cluster/ctxt" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/tui/progress" ) @@ -30,7 +30,7 @@ type StepDisplay struct { inner Task prefix string children map[Task]struct{} - DisplayMode log.DisplayMode + Logger *logprinter.Logger progressBar progress.Bar } @@ -56,14 +56,14 @@ func addChildren(m map[Task]struct{}, task Task) { } } -func newStepDisplay(prefix string, inner Task, m log.DisplayMode) *StepDisplay { +func newStepDisplay(prefix string, inner Task, logger *logprinter.Logger) *StepDisplay { children := make(map[Task]struct{}) addChildren(children, inner) return &StepDisplay{ inner: inner, prefix: prefix, children: children, - DisplayMode: m, + Logger: logger, progressBar: progress.NewSingleBar(prefix), } } @@ -74,9 +74,9 @@ func (s *StepDisplay) SetHidden(h bool) *StepDisplay { return s } -// SetDisplayMode set the value of log.DisplayMode flag -func (s *StepDisplay) SetDisplayMode(m log.DisplayMode) *StepDisplay { - s.DisplayMode = m +// SetLogger set the logger of step +func (s *StepDisplay) SetLogger(logger *logprinter.Logger) *StepDisplay { + s.Logger = logger return s } @@ -95,9 +95,9 @@ func (s *StepDisplay) Execute(ctx context.Context) error { return err } - switch s.DisplayMode { - case log.DisplayModeJSON, - log.DisplayModePlain: + switch s.Logger.GetDisplayMode() { + case logprinter.DisplayModeJSON, + logprinter.DisplayModePlain: break default: if singleBar, ok := s.progressBar.(*progress.SingleBar); ok { @@ -125,11 +125,11 @@ func (s *StepDisplay) Execute(ctx context.Context) error { } } - switch s.DisplayMode { - case log.DisplayModeJSON: + switch s.Logger.GetDisplayMode() { + case logprinter.DisplayModeJSON: _ = printDpJSON(dp) - case log.DisplayModePlain: - printDpPlain(dp) + case logprinter.DisplayModePlain: + printDpPlain(s.Logger, dp) default: s.progressBar.UpdateDisplay(dp) } @@ -154,11 +154,11 @@ func (s *StepDisplay) handleTaskBegin(task Task) { Prefix: s.prefix, Suffix: strings.Split(task.String(), "\n")[0], } - switch s.DisplayMode { - case log.DisplayModeJSON: + switch s.Logger.GetDisplayMode() { + case logprinter.DisplayModeJSON: _ = printDpJSON(dp) - case log.DisplayModePlain: - printDpPlain(dp) + case logprinter.DisplayModePlain: + printDpPlain(s.Logger, dp) default: s.progressBar.UpdateDisplay(dp) } @@ -172,11 +172,11 @@ func (s *StepDisplay) handleTaskProgress(task Task, p string) { Prefix: s.prefix, Suffix: strings.Split(p, "\n")[0], } - switch s.DisplayMode { - case log.DisplayModeJSON: + switch s.Logger.GetDisplayMode() { + case logprinter.DisplayModeJSON: _ = printDpJSON(dp) - case log.DisplayModePlain: - printDpPlain(dp) + case logprinter.DisplayModePlain: + printDpPlain(s.Logger, dp) default: s.progressBar.UpdateDisplay(dp) } @@ -187,7 +187,7 @@ func (s *StepDisplay) handleTaskProgress(task Task, p string) { type ParallelStepDisplay struct { inner *Parallel prefix string - DisplayMode log.DisplayMode + Logger *logprinter.Logger progressBar *progress.MultiBar } @@ -207,17 +207,17 @@ func newParallelStepDisplay(prefix string, ignoreError bool, sdTasks ...*StepDis } } -// SetDisplayMode set the value of log.DisplayMode flag -func (ps *ParallelStepDisplay) SetDisplayMode(m log.DisplayMode) *ParallelStepDisplay { - ps.DisplayMode = m +// SetLogger set the logger of step +func (ps *ParallelStepDisplay) SetLogger(logger *logprinter.Logger) *ParallelStepDisplay { + ps.Logger = logger return ps } // Execute implements the Task interface func (ps *ParallelStepDisplay) Execute(ctx context.Context) error { - switch ps.DisplayMode { - case log.DisplayModeJSON, - log.DisplayModePlain: + switch ps.Logger.GetDisplayMode() { + case logprinter.DisplayModeJSON, + logprinter.DisplayModePlain: break default: ps.progressBar.StartRenderLoop() @@ -246,11 +246,11 @@ func printDpJSON(dp *progress.DisplayProps) error { return nil } -func printDpPlain(dp *progress.DisplayProps) { +func printDpPlain(logger *logprinter.Logger, dp *progress.DisplayProps) { switch dp.Mode { case progress.ModeError: - log.Errorf("progress: %s", dp) + logger.Errorf("progress: %s", dp) default: - log.Infof("progress: %s", dp) + logger.Infof("progress: %s", dp) } } diff --git a/pkg/cluster/task/task.go b/pkg/cluster/task/task.go index 18d4442fae..3838a9a32e 100644 --- a/pkg/cluster/task/task.go +++ b/pkg/cluster/task/task.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/tiup/pkg/checkpoint" "github.com/pingcap/tiup/pkg/cluster/ctxt" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" ) var ( @@ -78,7 +78,8 @@ func (s *Serial) Execute(ctx context.Context) error { for _, t := range s.inner { if !isDisplayTask(t) { if !s.hideDetailDisplay { - log.Infof("+ [ Serial ] - %s", t.String()) + ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger). + Infof("+ [ Serial ] - %s", t.String()) } } ctxt.GetInner(ctx).Ev.PublishTaskBegin(t) @@ -135,7 +136,8 @@ func (pt *Parallel) Execute(ctx context.Context) error { }() if !isDisplayTask(t) { if !pt.hideDetailDisplay { - log.Infof("+ [Parallel] - %s", t.String()) + ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger). + Infof("+ [Parallel] - %s", t.String()) } } ctxt.GetInner(ctx).Ev.PublishTaskBegin(t) diff --git a/pkg/environment/env.go b/pkg/environment/env.go index b0b808cb16..56627864b4 100644 --- a/pkg/environment/env.go +++ b/pkg/environment/env.go @@ -23,10 +23,10 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/utils" + "go.uber.org/zap" "golang.org/x/mod/semver" ) @@ -107,7 +107,7 @@ func InitEnv(options repository.Options) (*Environment, error) { } v1repo = repository.NewV1Repo(mirror, options, local) - log.Verbose("Initialize repository finished in %s", time.Since(initRepo)) + zap.L().Debug("Initialize repository finished", zap.Duration("duration", time.Since(initRepo))) return &Environment{profile, v1repo}, nil } diff --git a/pkg/logger/audit.go b/pkg/logger/audit.go index 90c1c8de07..1b00f05beb 100644 --- a/pkg/logger/audit.go +++ b/pkg/logger/audit.go @@ -17,7 +17,7 @@ import ( "bytes" "github.com/pingcap/tiup/pkg/cluster/audit" - utils2 "github.com/pingcap/tiup/pkg/utils" + "github.com/pingcap/tiup/pkg/utils" "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -50,7 +50,7 @@ func OutputAuditLogIfEnabled() error { return nil } - if err := utils2.CreateDir(auditDir); err != nil { + if err := utils.CreateDir(auditDir); err != nil { return err } diff --git a/pkg/logger/log/log.go b/pkg/logger/printer/log.go similarity index 78% rename from pkg/logger/log/log.go rename to pkg/logger/printer/log.go index 65f658e335..441f30c2ac 100644 --- a/pkg/logger/log/log.go +++ b/pkg/logger/printer/log.go @@ -11,13 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package log +package logprinter import ( "encoding/json" "fmt" "io" "os" + "strings" "go.uber.org/zap" ) @@ -39,18 +40,21 @@ const ( DisplayModeJSON // JSON ) -// SetDisplayMode changes the global output format of logger -func SetDisplayMode(m DisplayMode) { - outputFmt = m -} - -// GetDisplayMode returns the current global output format -func GetDisplayMode() DisplayMode { - return outputFmt +func fmtDisplayMode(m string) DisplayMode { + var dp DisplayMode + switch strings.ToLower(m) { + case "json": + dp = DisplayModeJSON + case "plain", "text": + dp = DisplayModePlain + default: + dp = DisplayModeDefault + } + return dp } -func printLog(w io.Writer, level, format string, args ...interface{}) { - switch outputFmt { +func printLog(w io.Writer, mode DisplayMode, level, format string, args ...interface{}) { + switch mode { case DisplayModeJSON: obj := struct { Level string `json:"level"` @@ -70,6 +74,21 @@ func printLog(w io.Writer, level, format string, args ...interface{}) { } } +// SetDisplayMode changes the global output format of logger +func SetDisplayMode(m DisplayMode) { + outputFmt = m +} + +// GetDisplayMode returns the current global output format +func GetDisplayMode() DisplayMode { + return outputFmt +} + +// SetDisplayModeFromString changes the global output format of logger +func SetDisplayModeFromString(m string) { + outputFmt = fmtDisplayMode(m) +} + // Debugf output the debug message to console func Debugf(format string, args ...interface{}) { zap.L().Debug(fmt.Sprintf(format, args...)) @@ -79,21 +98,21 @@ func Debugf(format string, args ...interface{}) { // Deprecated: Use zap.L().Info() instead func Infof(format string, args ...interface{}) { zap.L().Info(fmt.Sprintf(format, args...)) - printLog(stdout, "info", format, args...) + printLog(stdout, outputFmt, "info", format, args...) } // Warnf output the warning message to console // Deprecated: Use zap.L().Warn() instead func Warnf(format string, args ...interface{}) { zap.L().Warn(fmt.Sprintf(format, args...)) - printLog(stderr, "warn", format, args...) + printLog(stderr, outputFmt, "warn", format, args...) } // Errorf output the error message to console // Deprecated: Use zap.L().Error() instead func Errorf(format string, args ...interface{}) { zap.L().Error(fmt.Sprintf(format, args...)) - printLog(stderr, "error", format, args...) + printLog(stderr, outputFmt, "error", format, args...) } // SetStdout redirect stdout to a custom writer diff --git a/pkg/logger/printer/logger.go b/pkg/logger/printer/logger.go new file mode 100644 index 0000000000..c298b0b88a --- /dev/null +++ b/pkg/logger/printer/logger.go @@ -0,0 +1,96 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logprinter + +import ( + "fmt" + "io" + "os" + + "go.uber.org/zap" +) + +// ContextKey is key used to store values in context +type ContextKey string + +// ContextKeyLogger is the key used for logger stored in context +const ContextKeyLogger ContextKey = "logger" + +// Logger is a set of fuctions writing output to custom writters, but still +// using the global zap logger as our default config does not writes everything +// to a memory buffer. +// TODO: use also separate zap loggers +type Logger struct { + outputFmt DisplayMode // output format of logger + + stdout io.Writer + stderr io.Writer +} + +// NewLogger creates a Logger with default settings +func NewLogger(m string) *Logger { + return &Logger{ + stdout: os.Stdout, + stderr: os.Stderr, + outputFmt: fmtDisplayMode(m), + } +} + +// SetStdout redirect stdout to a custom writer +func (l *Logger) SetStdout(w io.Writer) { + l.stdout = w +} + +// SetStderr redirect stderr to a custom writer +func (l *Logger) SetStderr(w io.Writer) { + l.stderr = w +} + +// SetDisplayMode changes the output format of logger +func (l *Logger) SetDisplayMode(m DisplayMode) { + l.outputFmt = m +} + +// SetDisplayModeFromString changes the output format of logger +func (l *Logger) SetDisplayModeFromString(m string) { + l.outputFmt = fmtDisplayMode(m) +} + +// GetDisplayMode returns the current output format +func (l *Logger) GetDisplayMode() DisplayMode { + return l.outputFmt +} + +// Debugf output the debug message to console +func (l *Logger) Debugf(format string, args ...interface{}) { + zap.L().Debug(fmt.Sprintf(format, args...)) +} + +// Infof output the log message to console +func (l *Logger) Infof(format string, args ...interface{}) { + zap.L().Info(fmt.Sprintf(format, args...)) + printLog(l.stdout, l.outputFmt, "info", format, args...) +} + +// Warnf output the warning message to console +func (l *Logger) Warnf(format string, args ...interface{}) { + zap.L().Warn(fmt.Sprintf(format, args...)) + printLog(l.stderr, l.outputFmt, "warn", format, args...) +} + +// Errorf output the error message to console +func (l *Logger) Errorf(format string, args ...interface{}) { + zap.L().Error(fmt.Sprintf(format, args...)) + printLog(l.stderr, l.outputFmt, "error", format, args...) +} diff --git a/pkg/logger/log/verbose.go b/pkg/logger/printer/verbose.go similarity index 80% rename from pkg/logger/log/verbose.go rename to pkg/logger/printer/verbose.go index a1d49e196b..69791e8258 100644 --- a/pkg/logger/log/verbose.go +++ b/pkg/logger/printer/verbose.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package log +package logprinter import ( "fmt" @@ -33,3 +33,11 @@ func Verbose(format string, args ...interface{}) { } fmt.Fprintln(stderr, "Verbose:", fmt.Sprintf(format, args...)) } + +// Verbose logs verbose messages +func (l *Logger) Verbose(format string, args ...interface{}) { + if !verbose { + return + } + fmt.Fprintln(l.stderr, "Verbose:", fmt.Sprintf(format, args...)) +} diff --git a/pkg/proxy/http_proxy.go b/pkg/proxy/http_proxy.go index a541790ebe..eb87834b06 100644 --- a/pkg/proxy/http_proxy.go +++ b/pkg/proxy/http_proxy.go @@ -24,6 +24,7 @@ import ( "github.com/appleboy/easyssh-proxy" perrs "github.com/pingcap/errors" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "go.uber.org/zap" "golang.org/x/crypto/ssh" ) @@ -34,10 +35,15 @@ type HTTPProxy struct { config *easyssh.MakeConfig l sync.RWMutex tr *http.Transport + logger *logprinter.Logger } // NewHTTPProxy creates and initializes a new http proxy -func NewHTTPProxy(host string, port int, user, password, keyFile, passphrase string) *HTTPProxy { +func NewHTTPProxy(host string, + port int, + user, password, keyFile, passphrase string, + logger *logprinter.Logger, +) *HTTPProxy { p := &HTTPProxy{ config: &easyssh.MakeConfig{ Server: host, @@ -45,6 +51,7 @@ func NewHTTPProxy(host string, port int, user, password, keyFile, passphrase str User: user, Timeout: 10 * time.Second, }, + logger: logger, } if len(keyFile) > 0 { diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index 02a7be451f..e83df39df6 100644 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -21,7 +21,7 @@ import ( "sync/atomic" "github.com/fatih/color" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" ) @@ -32,7 +32,14 @@ var ( ) // MaybeStartProxy maybe starts an inner http/tcp proxies -func MaybeStartProxy(host string, port int, user string, usePass bool, identity string) error { +func MaybeStartProxy( + host string, + port int, + user string, + usePass bool, + identity string, + logger *logprinter.Logger, +) error { if len(host) == 0 { return nil } @@ -58,13 +65,14 @@ func MaybeStartProxy(host string, port int, user string, usePass bool, identity sshProps.Password, sshProps.IdentityFile, sshProps.IdentityFilePassphrase, + logger, ), } - log.Infof(color.HiGreenString("Start HTTP inner proxy %s", httpProxy.Addr)) + logger.Infof(color.HiGreenString("Start HTTP inner proxy %s", httpProxy.Addr)) go func() { if err := httpProxy.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Errorf("Failed to listen HTTP proxy: %v", err) + logger.Errorf("Failed to listen HTTP proxy: %v", err) } }() @@ -72,10 +80,12 @@ func MaybeStartProxy(host string, port int, user string, usePass bool, identity host, port, user, sshProps.Password, sshProps.IdentityFile, - sshProps.IdentityFilePassphrase) + sshProps.IdentityFilePassphrase, + logger, + ) tcpProxy.Store(p) - log.Infof(color.HiGreenString("Start TCP inner proxy %s", p.endpoint)) + logger.Infof(color.HiGreenString("Start TCP inner proxy %s", p.endpoint)) return nil } diff --git a/pkg/proxy/tcp_proxy.go b/pkg/proxy/tcp_proxy.go index efb62c035c..d758c06506 100644 --- a/pkg/proxy/tcp_proxy.go +++ b/pkg/proxy/tcp_proxy.go @@ -24,7 +24,7 @@ import ( "github.com/appleboy/easyssh-proxy" perrs "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/utils" "go.uber.org/zap" "golang.org/x/crypto/ssh" @@ -39,10 +39,16 @@ type TCPProxy struct { config *easyssh.MakeConfig closed int32 endpoint string + logger *logprinter.Logger } // NewTCPProxy starts a 1to1 TCP proxy -func NewTCPProxy(host string, port int, user, password, keyFile, passphrase string) *TCPProxy { +func NewTCPProxy( + host string, + port int, + user, password, keyFile, passphrase string, + logger *logprinter.Logger, +) *TCPProxy { p := &TCPProxy{ config: &easyssh.MakeConfig{ Server: host, @@ -50,6 +56,7 @@ func NewTCPProxy(host string, port int, user, password, keyFile, passphrase stri User: user, Timeout: 10 * time.Second, }, + logger: logger, } if len(keyFile) > 0 { @@ -61,14 +68,14 @@ func NewTCPProxy(host string, port int, user, password, keyFile, passphrase stri port, err := utils.GetFreePort("127.0.0.1", 22345) if err != nil { - log.Errorf("get free port error: %v", err) + logger.Errorf("get free port error: %v", err) return nil } p.endpoint = fmt.Sprintf("127.0.0.1:%d", port) listener, err := net.Listen("tcp", p.endpoint) if err != nil { - log.Errorf("net.Listen error: %v", err) + logger.Errorf("net.Listen error: %v", err) return nil } p.listener = listener @@ -102,7 +109,7 @@ func (p *TCPProxy) Run(upstream []string) chan struct{} { if atomic.LoadInt32(&p.closed) == 1 { break FOR_LOOP } - log.Errorf("tcp proxy accept error: %v", err) + p.logger.Errorf("tcp proxy accept error: %v", err) continue FOR_LOOP } go p.forward(localConn, upstream) diff --git a/pkg/repository/mirror.go b/pkg/repository/mirror.go index 5fcfab79bd..2e5423c2d2 100644 --- a/pkg/repository/mirror.go +++ b/pkg/repository/mirror.go @@ -32,7 +32,7 @@ import ( "github.com/google/uuid" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/crypto/rand" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/repository/model" "github.com/pingcap/tiup/pkg/repository/store" "github.com/pingcap/tiup/pkg/repository/v1manifest" @@ -289,7 +289,7 @@ func (l *httpMirror) Open() error { func (l *httpMirror) download(url string, to string, maxSize int64) (io.ReadCloser, error) { defer func(start time.Time) { - log.Verbose("Download resource %s in %s", url, time.Since(start)) + logprinter.Verbose("Download resource %s in %s", url, time.Since(start)) }(time.Now()) client := grab.NewClient() @@ -486,7 +486,7 @@ func (l *httpMirror) Download(resource, targetDir string) error { _ = utils.Retry(func() error { var r io.ReadCloser if err != nil && l.isRetryable(err) { - log.Warnf("failed to download %s(%s), retrying...", resource, err.Error()) + logprinter.Warnf("failed to download %s(%s), retrying...", resource, err.Error()) } if r, err = l.download(l.prepareURL(resource), tmpFilePath, 0); err != nil { if l.isRetryable(err) { diff --git a/pkg/repository/model/model.go b/pkg/repository/model/model.go index 5cebeda874..5f900c0900 100644 --- a/pkg/repository/model/model.go +++ b/pkg/repository/model/model.go @@ -20,12 +20,12 @@ import ( cjson "github.com/gibson042/canonicaljson-go" "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/repository/store" "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/utils" "github.com/pingcap/tiup/pkg/version" + "go.uber.org/zap" ) // Backend defines operations on the manifests @@ -343,7 +343,11 @@ func (m *model) updateComponentManifest(manifest *v1manifest.Manifest) error { snapSigned := snap.Signed.(*v1manifest.Snapshot) lastVersion := snapSigned.Meta["/"+signed.Filename()].Version if signed.Version != lastVersion+1 { - log.Debugf("Component version not expected, expect %d, got %d", lastVersion+1, signed.Version) + zap.L().Debug( + "Component version not expected", + zap.Uint("expected", lastVersion+1), + zap.Uint("got", signed.Version), + ) return ErrorConflict } return m.txn.WriteManifest(fmt.Sprintf("%d.%s.json", signed.Version, signed.ID), manifest) diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 6fae4cfd02..f57a2a9354 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -32,7 +32,7 @@ import ( cjson "github.com/gibson042/canonicaljson-go" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/utils" "golang.org/x/mod/semver" @@ -203,7 +203,7 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error { // ensureManifests ensures that the snapshot, root, and index manifests are up to date and saved in r.local. func (r *V1Repository) ensureManifests() error { defer func(start time.Time) { - log.Verbose("Ensure manifests finished in %s", time.Since(start)) + logprinter.Verbose("Ensure manifests finished in %s", time.Since(start)) }(time.Now()) // Update snapshot. @@ -218,7 +218,7 @@ func (r *V1Repository) ensureManifests() error { // Postcondition: if returned error is nil, then the local snapshot and timestamp are up to date and return the snapshot func (r *V1Repository) updateLocalSnapshot() (*v1manifest.Snapshot, error) { defer func(start time.Time) { - log.Verbose("Update local snapshot finished in %s", time.Since(start)) + logprinter.Verbose("Update local snapshot finished in %s", time.Since(start)) }(time.Now()) timestampChanged, tsManifest, err := r.fetchTimestamp() @@ -308,7 +308,7 @@ func FnameWithVersion(fname string, version uint) string { func (r *V1Repository) updateLocalRoot() error { defer func(start time.Time) { - log.Verbose("Update local root finished in %s", time.Since(start)) + logprinter.Verbose("Update local root finished in %s", time.Since(start)) }(time.Now()) oldRoot, err := r.loadRoot() @@ -372,7 +372,7 @@ func (r *V1Repository) updateLocalRoot() error { // Precondition: the root manifest has been updated if necessary. func (r *V1Repository) updateLocalIndex(snapshot *v1manifest.Snapshot) error { defer func(start time.Time) { - log.Verbose("Update local index finished in %s", time.Since(start)) + logprinter.Verbose("Update local index finished in %s", time.Since(start)) }(time.Now()) // Update index (if needed). @@ -414,7 +414,7 @@ func (r *V1Repository) updateLocalIndex(snapshot *v1manifest.Snapshot) error { // Precondition: the snapshot and index manifests exist and are up to date. func (r *V1Repository) updateComponentManifest(id string, withYanked bool) (*v1manifest.Component, error) { defer func(start time.Time) { - log.Verbose("update component '%s' manifest finished in %s", id, time.Since(start)) + logprinter.Verbose("update component '%s' manifest finished in %s", id, time.Since(start)) }(time.Now()) // Find the component's entry in the index and snapshot manifests. @@ -527,7 +527,7 @@ func (r *V1Repository) fetchTimestamp() (changed bool, manifest *v1manifest.Mani } defer func(start time.Time) { - log.Verbose("Fetch timestamp finished in %s", time.Since(start)) + logprinter.Verbose("Fetch timestamp finished in %s", time.Since(start)) r.timestamp = manifest }(time.Now()) diff --git a/pkg/tui/progress/display_props.go b/pkg/tui/progress/display_props.go index 65ead9d217..48dd5d2462 100644 --- a/pkg/tui/progress/display_props.go +++ b/pkg/tui/progress/display_props.go @@ -38,7 +38,7 @@ func (m Mode) MarshalJSON() ([]byte, error) { var s string switch m { case ModeSpinner: - s = "spinner" + s = "running" case ModeProgress: s = "progress" case ModeDone: @@ -58,7 +58,7 @@ func (m *Mode) UnmarshalJSON(b []byte) error { return err } switch strings.ToLower(s) { - case "spinner": + case "spinner", "running": // keep "spinner" for compatiability *m = ModeSpinner case "progress": *m = ModeProgress @@ -78,7 +78,7 @@ func (m Mode) String() string { var s string switch m { case ModeSpinner: - s = "spinner" + s = "running" case ModeProgress: s = "progress" case ModeDone: diff --git a/pkg/utils/user.go b/pkg/utils/user.go index 6d8b11b2e7..974bf9324e 100644 --- a/pkg/utils/user.go +++ b/pkg/utils/user.go @@ -3,14 +3,14 @@ package utils import ( "os/user" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" ) // CurrentUser returns current login user func CurrentUser() string { user, err := user.Current() if err != nil { - log.Errorf("Get current user: %s", err) + logprinter.Errorf("Get current user: %s", err) return "root" } return user.Username @@ -20,7 +20,7 @@ func CurrentUser() string { func UserHome() string { user, err := user.Current() if err != nil { - log.Errorf("Get current user home: %s", err) + logprinter.Errorf("Get current user home: %s", err) return "root" } return user.HomeDir diff --git a/pkg/version/version.go b/pkg/version/version.go index 04dd32f763..8c1ecb6e1a 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -21,7 +21,7 @@ var ( // TiUPVerMajor is the major version of TiUP TiUPVerMajor = 1 // TiUPVerMinor is the minor version of TiUP - TiUPVerMinor = 7 + TiUPVerMinor = 8 // TiUPVerPatch is the patch version of TiUP TiUPVerPatch = 0 // TiUPVerName is an alternative name of the version diff --git a/server/handler/component.go b/server/handler/component.go index 687fa8fa46..6ce62c1376 100644 --- a/server/handler/component.go +++ b/server/handler/component.go @@ -19,7 +19,7 @@ import ( "github.com/gorilla/mux" "github.com/pingcap/fn" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/repository/model" "github.com/pingcap/tiup/pkg/repository/v1manifest" @@ -74,20 +74,20 @@ func (h *componentSigner) sign(r *http.Request, m *v1manifest.RawManifest) (sr * } } - log.Infof("Sign component manifest for %s, sid: %s", name, sid) + logprinter.Infof("Sign component manifest for %s, sid: %s", name, sid) if fileName, readCloser, err := h.sm.Read(sid); err == nil { info.ComponentData = &model.TarInfo{ Reader: readCloser, Name: fileName, } } else { - log.Errorf("Read tar info for component %s, sid: %s", name, sid) + logprinter.Errorf("Read tar info for component %s, sid: %s", name, sid) return nil, ErrorInternalError } comp := v1manifest.Component{} if err := json.Unmarshal(m.Signed, &comp); err != nil { - log.Errorf("Unmarshal manifest %s", err.Error()) + logprinter.Errorf("Unmarshal manifest %s", err.Error()) return nil, ErrorInvalidManifest } @@ -102,13 +102,13 @@ func (h *componentSigner) sign(r *http.Request, m *v1manifest.RawManifest) (sr * case model.ErrorWrongSignature: return nil, ErrorForbiden case model.ErrorWrongChecksum, model.ErrorWrongFileName: - log.Errorf("Publish component: %s", err.Error()) + logprinter.Errorf("Publish component: %s", err.Error()) return nil, ErrorInvalidTarball case nil: return nil, nil default: h.sm.Delete(sid) - log.Errorf("Publish component: %s", err.Error()) + logprinter.Errorf("Publish component: %s", err.Error()) return nil, ErrorInternalError } } diff --git a/server/handler/handler.go b/server/handler/handler.go index 85b4902f37..5876c64daa 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -17,7 +17,7 @@ import ( "context" "github.com/pingcap/fn" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" ) // errorMessage is used for error response @@ -28,15 +28,15 @@ type errorMessage struct { func init() { fn.SetErrorEncoder(func(ctx context.Context, err error) interface{} { - log.Debugf("Response an error message to client") + logprinter.Debugf("Response an error message to client") if e, ok := err.(statusError); ok { - log.Debugf("Response status error to client: %s", e.Error()) + logprinter.Debugf("Response status error to client: %s", e.Error()) return &errorMessage{ Status: e.Status(), Message: e.Error(), } } - log.Debugf("Unknow error occurred: %s", err.Error()) + logprinter.Debugf("Unknow error occurred: %s", err.Error()) return &errorMessage{ Status: "UNKNOWN_ERROR", Message: "make sure your request is valid", diff --git a/server/handler/rotate.go b/server/handler/rotate.go index 6917ede00a..e28fb3be94 100644 --- a/server/handler/rotate.go +++ b/server/handler/rotate.go @@ -18,7 +18,7 @@ import ( "net/http" "github.com/pingcap/fn" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/repository/model" "github.com/pingcap/tiup/pkg/repository/v1manifest" @@ -40,7 +40,7 @@ func (h *rootSigner) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *rootSigner) sign(m *v1manifest.RawManifest) (sr *simpleResponse, err statusError) { root := v1manifest.Root{} if err := json.Unmarshal(m.Signed, &root); err != nil { - log.Errorf("Unmarshal manifest %s", err.Error()) + logprinter.Errorf("Unmarshal manifest %s", err.Error()) return nil, ErrorInvalidManifest } @@ -55,7 +55,7 @@ func (h *rootSigner) sign(m *v1manifest.RawManifest) (sr *simpleResponse, err st case nil: return nil, nil default: - log.Errorf("Rotate root manifest: %s", err.Error()) + logprinter.Errorf("Rotate root manifest: %s", err.Error()) return nil, ErrorInternalError } } diff --git a/server/handler/tarball.go b/server/handler/tarball.go index bc9fa20e82..e98feda0a0 100644 --- a/server/handler/tarball.go +++ b/server/handler/tarball.go @@ -18,7 +18,7 @@ import ( "github.com/gorilla/mux" "github.com/pingcap/fn" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/server/session" ) @@ -40,7 +40,7 @@ func (h *tarballUploader) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *tarballUploader) upload(r *http.Request) (*simpleResponse, statusError) { sid := mux.Vars(r)["sid"] - log.Infof("Uploading tarball, sid: %s", sid) + logprinter.Infof("Uploading tarball, sid: %s", sid) if err := r.ParseMultipartForm(MaxMemory); err != nil { // TODO: log error here @@ -55,7 +55,7 @@ func (h *tarballUploader) upload(r *http.Request) (*simpleResponse, statusError) defer file.Close() if err := h.sm.Write(sid, handler.Filename, file); err != nil { - log.Errorf("Error to write tarball: %s", err.Error()) + logprinter.Errorf("Error to write tarball: %s", err.Error()) return nil, ErrorInternalError } diff --git a/server/main.go b/server/main.go index fb029fb3fa..bc7e5b13cd 100644 --- a/server/main.go +++ b/server/main.go @@ -17,7 +17,7 @@ import ( "fmt" "os" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/version" "github.com/spf13/cobra" ) @@ -49,6 +49,6 @@ func main() { cmd.Flags().StringVarP(&upstream, "upstream", "", upstream, "specify the upstream mirror") if err := cmd.Execute(); err != nil { - log.Errorf("Execute command: %s", err.Error()) + logprinter.Errorf("Execute command: %s", err.Error()) } } diff --git a/server/rotate/rotate_server.go b/server/rotate/rotate_server.go index b9ae91f59b..fc9bc6e6f7 100644 --- a/server/rotate/rotate_server.go +++ b/server/rotate/rotate_server.go @@ -11,7 +11,7 @@ import ( "github.com/gorilla/mux" "github.com/pingcap/errors" "github.com/pingcap/fn" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/tui/progress" ) @@ -80,7 +80,7 @@ func Serve(addr string, root *v1manifest.Root) (*v1manifest.Manifest, error) { srv := &http.Server{Addr: addr, Handler: r} go func() { if err := srv.ListenAndServe(); err != nil { - log.Errorf("server closed: %s", err.Error()) + logprinter.Errorf("server closed: %s", err.Error()) } close(sigCh) }() diff --git a/server/router.go b/server/router.go index f00c0398c5..b9509a8b73 100644 --- a/server/router.go +++ b/server/router.go @@ -18,7 +18,7 @@ import ( "time" "github.com/gorilla/mux" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/server/handler" ) @@ -34,11 +34,11 @@ func (w *traceResponseWriter) WriteHeader(code int) { func httpRequestMiddleware(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - log.Infof("Request : %s - %s - %s", r.RemoteAddr, r.Method, r.URL) + logprinter.Infof("Request : %s - %s - %s", r.RemoteAddr, r.Method, r.URL) start := time.Now() tw := &traceResponseWriter{w, http.StatusOK} h.ServeHTTP(tw, r) - log.Infof("Response [%d] : %s - %s - %s (%.3f sec)", + logprinter.Infof("Response [%d] : %s - %s - %s (%.3f sec)", tw.statusCode, r.RemoteAddr, r.Method, r.URL, time.Since(start).Seconds()) }) } diff --git a/server/session/session.go b/server/session/session.go index 41f184d7f1..087845b6e5 100644 --- a/server/session/session.go +++ b/server/session/session.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" ) // Max alive time of a session @@ -56,7 +56,7 @@ func (s *sessionManager) Write(id, name string, reader io.Reader) error { if _, ok := s.m.Load(id); ok { return ErrorSessionConflict } - log.Debugf("Begin new session: %s", id) + logprinter.Debugf("Begin new session: %s", id) s.m.Store(id, name) go s.gc(id) @@ -113,7 +113,7 @@ func (s *sessionManager) Read(id string) (string, io.ReadCloser, error) { // Delele delete a session func (s *sessionManager) Delete(id string) { - log.Debugf("Delete session: %s", id) + logprinter.Debugf("Delete session: %s", id) n, ok := s.m.Load(id) if !ok { return diff --git a/server/static.go b/server/static.go index 94d00be615..c8d4bd6c92 100644 --- a/server/static.go +++ b/server/static.go @@ -20,7 +20,7 @@ import ( "os" "path" - "github.com/pingcap/tiup/pkg/logger/log" + logprinter "github.com/pingcap/tiup/pkg/logger/printer" ) // staticServer start a static web server @@ -32,10 +32,10 @@ func staticServer(local string, upstream string) http.Handler { f.Close() } else if os.IsNotExist(err) && upstream != "" { if err := proxyUpstream(w, r, path.Join(local, path.Clean(r.URL.Path)), upstream); err != nil { - log.Errorf("Proxy upstream: %s", err.Error()) + logprinter.Errorf("Proxy upstream: %s", err.Error()) fsh.ServeHTTP(w, r) } - log.Errorf("Handle file: %s", err.Error()) + logprinter.Errorf("Handle file: %s", err.Error()) return } fsh.ServeHTTP(w, r) diff --git a/tests/tiup-dm/script/task/run.sh b/tests/tiup-dm/script/task/run.sh index f38a0550ec..66734dfeae 100755 --- a/tests/tiup-dm/script/task/run.sh +++ b/tests/tiup-dm/script/task/run.sh @@ -9,6 +9,7 @@ set -eu pushd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +tiup install dmctl:nightly ctl="tiup dmctl:nightly" wd=$(pwd) diff --git a/tests/tiup/test_tiup.sh b/tests/tiup/test_tiup.sh index 8abe006163..f04fc7ffdf 100755 --- a/tests/tiup/test_tiup.sh +++ b/tests/tiup/test_tiup.sh @@ -1,20 +1,9 @@ #!/usr/bin/env bash -set -eu +set -eux TEST_DIR=$(cd "$(dirname "$0")"; pwd) -TMP_DIR=$TEST_DIR/_tmp - -rm -rf $TMP_DIR/data - -# Profile home directory -mkdir -p $TMP_DIR/home/bin/ -export TIUP_HOME=$TMP_DIR/home -curl https://tiup-mirrors.pingcap.com/root.json -o $TMP_DIR/home/bin/root.json - -# Prepare data directory -mkdir -p $TMP_DIR/data -export TIUP_INSTANCE_DATA_DIR=$TMP_DIR/data +TMP_DIR=`mktemp -d` mkdir -p $TEST_DIR/cover @@ -27,10 +16,17 @@ function tiup() { fi } +rm -rf $TMP_DIR/data + +# Profile home directory +mkdir -p $TMP_DIR/home/bin/ +export TIUP_HOME=$TMP_DIR/home +tiup mirror set --reset + tiup list tiup tiup help -tiup install tidb:v3.0.13 +tiup install tidb:v5.2.2 tiup update tidb tiup update tidb --nightly tiup --binary tidb:nightly @@ -41,30 +37,33 @@ tiup env TIUP_SSHPASS_PROMPT="password" tiup env TIUP_SSHPASS_PROMPT | grep password # test mirror -cat > /tmp/hello.sh << EOF +CMP_TMP_DIR=`mktemp -d` +cat > $CMP_TMP_DIR/hello.sh << EOF #! /bin/sh echo "hello, TiDB" EOF -chmod 755 /tmp/hello.sh -tar -C /tmp -czf /tmp/hello.tar.gz hello.sh +chmod 755 $CMP_TMP_DIR/hello.sh +tar -C $CMP_TMP_DIR -czf $CMP_TMP_DIR/hello.tar.gz hello.sh tiup mirror genkey -tiup mirror init /tmp/test-mirror-a -tiup mirror set /tmp/test-mirror-a +TEST_MIRROR_A=`mktemp -d` +tiup mirror init $TEST_MIRROR_A +tiup mirror set $TEST_MIRROR_A tiup mirror grant pingcap echo "should fail" ! tiup mirror grant pingcap # this should failed -tiup mirror publish hello v0.0.1 /tmp/hello.tar.gz hello.sh +tiup mirror publish hello v0.0.1 $CMP_TMP_DIR/hello.tar.gz hello.sh tiup hello:v0.0.1 | grep TiDB -tiup mirror init /tmp/test-mirror-b -tiup mirror set /tmp/test-mirror-b +TEST_MIRROR_B=`mktemp -d` +tiup mirror init $TEST_MIRROR_B +tiup mirror set $TEST_MIRROR_B tiup mirror grant pingcap -tiup mirror publish hello v0.0.2 /tmp/hello.tar.gz hello.sh -tiup mirror set /tmp/test-mirror-a -tiup mirror merge /tmp/test-mirror-b +tiup mirror publish hello v0.0.2 $CMP_TMP_DIR/hello.tar.gz hello.sh +tiup mirror set $TEST_MIRROR_A +tiup mirror merge $TEST_MIRROR_B tiup hello:v0.0.2 | grep TiDB tiup uninstall @@ -72,3 +71,5 @@ tiup uninstall tidb:v3.0.13 tiup uninstall tidb --all tiup uninstall --all tiup uninstall --self + +rm -rf $TMP_DIR $CMP_TMP_DIR $TEST_MIRROR_A $TEST_MIRROR_B From 90f3d355b89febebf587c1dab9150f72c179888a Mon Sep 17 00:00:00 2001 From: srstack Date: Tue, 7 Dec 2021 15:20:49 +0800 Subject: [PATCH 27/28] m --- pkg/cluster/manager/builder.go | 40 ++++++++-------------------------- pkg/cluster/manager/deploy.go | 11 +++++----- pkg/cluster/manager/tls.go | 22 +++++++++++-------- pkg/cluster/task/builder.go | 4 ++-- 4 files changed, 30 insertions(+), 47 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 7359e8e49d..26c90a66df 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -150,14 +150,13 @@ func buildScaleOutTask( // Download missing component - downloadCompTasks = convertStepDisplaysToTasks(buildDownloadCompTasks( + downloadCompTasks = buildDownloadCompTasks( base.Version, newPart, m.logger, gOpt, m.bindVersion, - )) - + ) sshType := topo.BaseTopo().GlobalOptions.SSHType @@ -238,34 +237,11 @@ func buildScaleOutTask( return nil, iterErr } - certificateTasks, err := buildCertificateTasks(m, name, newPart, base, gOpt, p) if err != nil { return nil, err } - hasImported := false - noAgentHosts := set.NewStringSet() - - mergedTopo.IterInstance(func(inst spec.Instance) { - deployDir := spec.Abs(base.User, inst.DeployDir()) - // data dir would be empty for components which don't need it - dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) - // log dir will always be with values, but might not used by the component - logDir := spec.Abs(base.User, inst.LogDir()) - - // Download and copy the latest component to remote if the cluster is imported from Ansible - tb := task.NewBuilder(m.logger) - if inst.IsImported() { - switch compName := inst.ComponentName(); compName { - case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: - version := m.bindVersion(compName, base.Version) - tb.Download(compName, inst.OS(), inst.Arch(), version). - CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) - } - hasImported = true - } - // init scale out config scaleOutConfigTasks := buildScaleConfigTasks(m, name, topo, newPart, base, gOpt, p) @@ -324,6 +300,7 @@ func buildScaleOutTask( noAgentHosts, *topo.BaseTopo().GlobalOptions, topo.GetMonitoredOptions(), + m.logger, gOpt.SSHTimeout, gOpt.OptTimeout, gOpt, @@ -402,7 +379,7 @@ func buildScaleConfigTasks( // log dir will always be with values, but might not used by the component logDir := spec.Abs(base.User, inst.LogDir()) - t := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + t := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). ScaleConfig( name, base.Version, @@ -548,7 +525,7 @@ func buildMonitoredCertificateTasks( tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) // Deploy component - tb := task.NewSimpleUerSSH(host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). + tb := task.NewSimpleUerSSH(m.logger, host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). Mkdir(globalOptions.User, host, tlsDir) if comp == spec.ComponentBlackboxExporter { @@ -777,7 +754,7 @@ func buildTLSTask( // load certificate file if topo.BaseTopo().GlobalOptions.TLSEnabled { tlsDir := m.specManager.Path(name, spec.TLSCertKeyDir) - log.Infof("Generate certificate: %s", color.YellowString(tlsDir)) + m.logger.Infof("Generate certificate: %s", color.YellowString(tlsDir)) if err := m.loadCertificate(name, topo.BaseTopo().GlobalOptions, reloadCertificate); err != nil { return nil, err } @@ -793,7 +770,7 @@ func buildTLSTask( // handle dir scheme changes if hasImported { if err := spec.HandleImportPathMigration(name); err != nil { - return task.NewBuilder(gOpt.DisplayMode).Build(), err + return task.NewBuilder(m.logger).Build(), err } } @@ -820,6 +797,7 @@ func buildTLSTask( noAgentHosts, *topo.BaseTopo().GlobalOptions, topo.GetMonitoredOptions(), + m.logger, gOpt.SSHTimeout, gOpt.OptTimeout, gOpt, @@ -882,7 +860,7 @@ func buildCertificateTasks( deployDir := spec.Abs(base.User, inst.DeployDir()) tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) - tb := task.NewSimpleUerSSH(inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + tb := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). Mkdir(base.User, inst.GetHost(), deployDir, tlsDir) ca, err := crypto.ReadCA( diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index aca61d864f..899259977f 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -228,8 +228,8 @@ func (m *Manager) Deploy( t := task.NewBuilder(m.logger). RootSSH( - inst.GetHost(), - inst.GetSSHPort(), + host, + hostInfo.ssh, opt.User, sshConnProps.Password, sshConnProps.IdentityFile, @@ -246,9 +246,9 @@ func (m *Manager) Deploy( gOpt.SSHType, globalOptions.SSHType, ). - EnvInit(inst.GetHost(), globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). - Mkdir(globalOptions.User, inst.GetHost(), dirs...). - BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", inst.GetHost(), inst.GetSSHPort())) + EnvInit(host, globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). + Mkdir(globalOptions.User, host, dirs...). + BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", host, hostInfo.ssh)) envInitTasks = append(envInitTasks, t) } @@ -384,6 +384,7 @@ func (m *Manager) Deploy( noAgentHosts, *topo.BaseTopo().GlobalOptions, topo.GetMonitoredOptions(), + m.logger, gOpt.SSHTimeout, gOpt.OptTimeout, gOpt, diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 0b3b2ad91d..4eeacc1106 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" "golang.org/x/mod/semver" @@ -56,11 +56,10 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica // if force is true, skip this check if globalOptions.TLSEnabled == enable && !gOpt.Force { if enable { - log.Infof("cluster `%s` TLS status is already enable", name) + return perrs.Errorf("cluster `%s` TLS status is already enable\n", name) } else { - log.Infof("cluster `%s` TLS status is already disable", name) + return perrs.Errorf("cluster `%s` TLS status is already disable\n", name) } - return nil } globalOptions.TLSEnabled = enable @@ -93,7 +92,12 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica return err } - if err := t.Execute(ctxt.New(context.Background(), gOpt.Concurrency)); err != nil { + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. return err @@ -110,13 +114,13 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica if cleanCertificate { os.RemoveAll(m.specManager.Path(name, spec.TLSCertKeyDir)) } - log.Infof("\tCleanup localhost tls file success") + m.logger.Infof("\tCleanup localhost tls file success") } if enable { - log.Infof("Enable cluster `%s` TLS between TiDB components successfully", name) + m.logger.Infof("Enable cluster `%s` TLS between TiDB components successfully", name) } else { - log.Infof("Disable cluster `%s` TLS between TiDB components successfully", name) + m.logger.Infof("Disable cluster `%s` TLS between TiDB components successfully", name) } return nil } @@ -164,7 +168,7 @@ func getTLSFileMap(m *Manager, clusterName string, topo spec.Topology, } } - log.Warnf("The parameter `%s` will delete the following files: %s", color.YellowString("--clean-certificate"), delFileList) + m.logger.Warnf("The parameter `%s` will delete the following files: %s", color.YellowString("--clean-certificate"), delFileList) if !skipConfirm { if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index 0a575cfdce..3eb7829d74 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -71,8 +71,8 @@ func (b *Builder) RootSSH( } // NewSimpleUerSSH append a UserSSH task to the current task collection with operator.Options and SSHConnectionProps -func NewSimpleUerSSH(host string, port int, user string, gOpt operator.Options, p *tui.SSHConnectionProps, sshType executor.SSHType) *Builder { - return NewBuilder(gOpt.DisplayMode). +func NewSimpleUerSSH(logger *logprinter.Logger, host string, port int, user string, gOpt operator.Options, p *tui.SSHConnectionProps, sshType executor.SSHType) *Builder { + return NewBuilder(logger). UserSSH( host, port, From 3f9d1402993e5c4d9a602406dd7893f24b63f32a Mon Sep 17 00:00:00 2001 From: srstack Date: Tue, 7 Dec 2021 17:16:12 +0800 Subject: [PATCH 28/28] perfect tips --- pkg/cluster/manager/builder.go | 50 +++++++++++++++++--------------- pkg/cluster/manager/tls.go | 5 ++-- pkg/cluster/operation/destroy.go | 2 +- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 438d415a80..1b9c84970a 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -55,7 +55,7 @@ func buildReloadPromTasks( } t := task.NewBuilder(logger). SystemCtl(inst.GetHost(), inst.ServiceName(), "reload", true). - BuildAsStep(fmt.Sprintf(" - Reload prometheus %s -> %s", inst.ComponentName(), inst.ID())) + BuildAsStep(fmt.Sprintf(" - Reload %s -> %s", inst.ComponentName(), inst.ID())) tasks = append(tasks, t) } return tasks @@ -218,11 +218,7 @@ func buildScaleOutTask( return nil, iterErr } - certificateTasks, err := buildCertificateTasks(m, name, newPart, base, gOpt, p) - if err != nil { - return nil, err - } - + // Download and copy the latest component to remote if the cluster is imported from Ansible mergedTopo.IterInstance(func(inst spec.Instance) { if inst.IsImported() { deployDir := spec.Abs(base.User, inst.DeployDir()) @@ -235,12 +231,18 @@ func buildScaleOutTask( tb.Download(compName, inst.OS(), inst.Arch(), version). CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) } + deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Deploy instance %s -> %s", inst.ComponentName(), inst.ID()))) } }) // init scale out config scaleOutConfigTasks := buildScaleConfigTasks(m, name, topo, newPart, base, gOpt, p) + certificateTasks, err := buildCertificateTasks(m, name, newPart, base, gOpt, p) + if err != nil { + return nil, err + } + // always ignore config check result in scale out gOpt.IgnoreConfigCheck = true refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, mergedTopo, base, gOpt, nil) @@ -272,36 +274,36 @@ func buildScaleOutTask( downloadCompTasks = append(downloadCompTasks, dlTasks...) deployCompTasks = append(deployCompTasks, dpTasks...) - // monitor tls file - moniterCertificateTasks, err := buildMonitoredCertificateTasks( - m, + // monitor config + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, name, uninitializedHosts, noAgentHosts, - topo.BaseTopo().GlobalOptions, + *topo.BaseTopo().GlobalOptions, topo.GetMonitoredOptions(), + m.logger, + gOpt.SSHTimeout, + gOpt.OptTimeout, gOpt, p, ) - if err != nil { - return nil, err - } - certificateTasks = append(certificateTasks, moniterCertificateTasks...) - // monitor config - monitorConfigTasks := buildInitMonitoredConfigTasks( - m.specManager, + // monitor tls file + moniterCertificateTasks, err := buildMonitoredCertificateTasks( + m, name, uninitializedHosts, noAgentHosts, - *topo.BaseTopo().GlobalOptions, + topo.BaseTopo().GlobalOptions, topo.GetMonitoredOptions(), - m.logger, - gOpt.SSHTimeout, - gOpt.OptTimeout, gOpt, p, ) + if err != nil { + return nil, err + } + certificateTasks = append(certificateTasks, moniterCertificateTasks...) builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) if err != nil { @@ -338,8 +340,8 @@ func buildScaleOutTask( builder.Func("Start new instances", func(ctx context.Context) error { return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg) }). - ParallelStep("+ Refresh conifg", gOpt.Force, refreshConfigTasks...). - ParallelStep("+ Reload Prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...) + ParallelStep("+ Refresh components conifgs", gOpt.Force, refreshConfigTasks...). + ParallelStep("+ Reload prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...) } // remove scale-out file lock @@ -586,7 +588,7 @@ func buildInitMonitoredConfigTasks( Cache: specManager.Path(name, spec.TempConfigPath), }, ). - BuildAsStep(fmt.Sprintf(" - Init config %s -> %s", comp, host)) + BuildAsStep(fmt.Sprintf(" - Generate config %s -> %s", comp, host)) tasks = append(tasks, t) } } diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go index 4eeacc1106..5e6eab2b14 100644 --- a/pkg/cluster/manager/tls.go +++ b/pkg/cluster/manager/tls.go @@ -56,10 +56,11 @@ func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertifica // if force is true, skip this check if globalOptions.TLSEnabled == enable && !gOpt.Force { if enable { - return perrs.Errorf("cluster `%s` TLS status is already enable\n", name) + m.logger.Infof("cluster `%s` TLS status is already enable\n", name) } else { - return perrs.Errorf("cluster `%s` TLS status is already disable\n", name) + m.logger.Infof("cluster `%s` TLS status is already disable\n", name) } + return nil } globalOptions.TLSEnabled = enable diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index 35e8f74386..a899852e7b 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -337,7 +337,7 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) e := ctxt.GetInner(ctx).Get(ins.GetHost()) - logger.Infof("Destroying instance %s", ins.GetHost()) + logger.Infof("\tDestroying instance %s", ins.GetHost()) var dataDirs []string if len(ins.DataDir()) > 0 {