diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go index 44fba20ef9..82ad9319db 100644 --- a/components/cluster/command/root.go +++ b/components/cluster/command/root.go @@ -210,6 +210,7 @@ func init() { newTelemetryCmd(), newReplayCmd(), newTemplateCmd(), + newTLSCmd(), ) } diff --git a/components/cluster/command/tls.go b/components/cluster/command/tls.go new file mode 100644 index 0000000000..c23067c740 --- /dev/null +++ b/components/cluster/command/tls.go @@ -0,0 +1,71 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "strings" + + perrs "github.com/pingcap/errors" + "github.com/spf13/cobra" +) + +func newTLSCmd() *cobra.Command { + var ( + reloadCertificate bool // reload certificate when the cluster enable encrypted communication + cleanCertificate bool // cleanup certificate when the cluster disable encrypted communication + enableTLS bool + ) + + cmd := &cobra.Command{ + Use: "tls ", + Short: "Enable/Disable TLS between TiDB components", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 2 { + return cmd.Help() + } + + if err := validRoles(gOpt.Roles); err != nil { + return err + } + clusterName := args[0] + clusterReport.ID = scrubClusterName(clusterName) + teleCommand = append(teleCommand, scrubClusterName(clusterName)) + + switch strings.ToLower(args[1]) { + case "enable": + enableTLS = true + case "disable": + enableTLS = false + default: + return perrs.New("enable or disable must be specified at least one") + } + + if enableTLS && cleanCertificate { + return perrs.New("clean-certificate only works when tls disable") + } + + if !enableTLS && reloadCertificate { + return perrs.New("reload-certificate only works when tls enable") + } + + return cm.TLS(clusterName, gOpt, enableTLS, cleanCertificate, reloadCertificate, skipConfirm) + }, + } + + cmd.Flags().BoolVar(&cleanCertificate, "clean-certificate", false, "Cleanup the certificate file if it already exists when tls disable") + cmd.Flags().BoolVar(&reloadCertificate, "reload-certificate", false, "Load the certificate file whether it exists or not when tls enable") + cmd.Flags().BoolVar(&gOpt.Force, "force", false, "Force enable/disable tls regardless of the current state") + + return cmd +} diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go index 8e9ded2f5f..11775a02ba 100644 --- a/components/dm/spec/logic.go +++ b/components/dm/spec/logic.go @@ -140,10 +140,21 @@ func (i *MasterInstance) InitConfig( return err } + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + specConfig := spec.Config return i.MergeServerConfig(ctx, e, i.topo.ServerConfigs.Master, specConfig, paths) } +// setTLSConfig set TLS Config to support enable/disable TLS +// MasterInstance no need to configure TLS +func (i *MasterInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // ScaleConfig deploy temporary config on scaling func (i *MasterInstance) ScaleConfig( ctx context.Context, @@ -271,10 +282,21 @@ func (i *WorkerInstance) InitConfig( return err } + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + specConfig := spec.Config return i.MergeServerConfig(ctx, e, i.topo.ServerConfigs.Worker, specConfig, paths) } +// setTLSConfig set TLS Config to support enable/disable TLS +// workrsInstance no need to configure TLS +func (i *WorkerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // ScaleConfig deploy temporary config on scaling func (i *WorkerInstance) ScaleConfig( ctx context.Context, @@ -322,7 +344,7 @@ func (topo *Specification) ComponentsByStartOrder() (comps []Component) { // "dm-master", "dm-worker" comps = append(comps, &DMMasterComponent{topo}) comps = append(comps, &DMWorkerComponent{topo}) - comps = append(comps, &spec.MonitorComponent{Topology: topo}) + comps = append(comps, &spec.MonitorComponent{Topology: topo}) // prometheus comps = append(comps, &spec.GrafanaComponent{Topology: topo}) comps = append(comps, &spec.AlertManagerComponent{Topology: topo}) return diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 4caf5390a5..1b9c84970a 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -19,6 +19,7 @@ import ( "path/filepath" "strings" + "github.com/fatih/color" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" @@ -37,7 +38,7 @@ func buildReloadPromTasks( logger *logprinter.Logger, gOpt operator.Options, nodes ...string, -) []task.Task { +) []*task.StepDisplay { monitor := spec.FindComponent(topo, spec.ComponentPrometheus) if monitor == nil { return nil @@ -46,7 +47,7 @@ func buildReloadPromTasks( if len(instances) == 0 { return nil } - var tasks []task.Task + var tasks []*task.StepDisplay deletedNodes := set.NewStringSet(nodes...) for _, inst := range monitor.Instances() { if deletedNodes.Exist(inst.ID()) { @@ -54,7 +55,7 @@ func buildReloadPromTasks( } t := task.NewBuilder(logger). SystemCtl(inst.GetHost(), inst.ServiceName(), "reload", true). - Build() + BuildAsStep(fmt.Sprintf(" - Reload %s -> %s", inst.ComponentName(), inst.ID())) tasks = append(tasks, t) } return tasks @@ -74,10 +75,9 @@ func buildScaleOutTask( final func(b *task.Builder, name string, meta spec.Metadata, gOpt operator.Options), ) (task.Task, error) { var ( - envInitTasks []task.Task // tasks which are used to initialize environment - downloadCompTasks []task.Task // tasks which are used to download components - deployCompTasks []task.Task // tasks which are used to copy components to remote host - refreshConfigTasks []task.Task // tasks which are used to refresh configuration + envInitTasks []*task.StepDisplay // tasks which are used to initialize environment + downloadCompTasks []*task.StepDisplay // tasks which are used to download components + deployCompTasks []*task.StepDisplay // tasks which are used to copy components to remote host ) topo := metadata.GetTopology() @@ -121,6 +121,7 @@ func buildScaleOutTask( dirs = append(dirs, spec.Abs(globalOptions.User, dirname)) } } + t := task.NewBuilder(m.logger). RootSSH( instance.GetHost(), @@ -143,18 +144,18 @@ func buildScaleOutTask( ). EnvInit(instance.GetHost(), base.User, base.Group, opt.SkipCreateUser || globalOptions.User == opt.User). Mkdir(globalOptions.User, instance.GetHost(), dirs...). - Build() + BuildAsStep(fmt.Sprintf(" - Initialized host %s ", host)) envInitTasks = append(envInitTasks, t) }) // Download missing component - downloadCompTasks = convertStepDisplaysToTasks(buildDownloadCompTasks( + downloadCompTasks = buildDownloadCompTasks( base.Version, newPart, m.logger, gOpt, m.bindVersion, - )) + ) sshType := topo.BaseTopo().GlobalOptions.SSHType @@ -174,27 +175,8 @@ func buildScaleOutTask( filepath.Join(deployDir, "conf"), filepath.Join(deployDir, "scripts"), } - if topo.BaseTopo().GlobalOptions.TLSEnabled { - deployDirs = append(deployDirs, filepath.Join(deployDir, "tls")) - } // Deploy component - tb := task.NewBuilder(m.logger). - UserSSH( - inst.GetHost(), - inst.GetSSHPort(), - base.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - sshType, - ). + tb := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, sshType). Mkdir(base.User, inst.GetHost(), deployDirs...). Mkdir(base.User, inst.GetHost(), dataDirs...). Mkdir(base.User, inst.GetHost(), logDir) @@ -228,91 +210,42 @@ func buildScaleOutTask( ) } } - // generate and transfer tls cert for instance - if topo.BaseTopo().GlobalOptions.TLSEnabled { - ca, err := crypto.ReadCA( - name, - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), - ) - if err != nil { - iterErr = err - return - } - tb = tb.TLSCert( - inst.GetHost(), - inst.ComponentName(), - inst.Role(), - inst.GetMainPort(), - ca, - meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) - } - t := tb.ScaleConfig(name, - base.Version, - m.specManager, - topo, - inst, - base.User, - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - }, - ).Build() - deployCompTasks = append(deployCompTasks, t) + deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Deploy instance %s -> %s", inst.ComponentName(), inst.ID()))) }) + if iterErr != nil { return nil, iterErr } - hasImported := false - noAgentHosts := set.NewStringSet() - + // Download and copy the latest component to remote if the cluster is imported from Ansible mergedTopo.IterInstance(func(inst spec.Instance) { - deployDir := spec.Abs(base.User, inst.DeployDir()) - // data dir would be empty for components which don't need it - dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) - // log dir will always be with values, but might not used by the component - logDir := spec.Abs(base.User, inst.LogDir()) - - // Download and copy the latest component to remote if the cluster is imported from Ansible - tb := task.NewBuilder(m.logger) if inst.IsImported() { + deployDir := spec.Abs(base.User, inst.DeployDir()) + // data dir would be empty for components which don't need it + // Download and copy the latest component to remote if the cluster is imported from Ansible + tb := task.NewBuilder(m.logger) switch compName := inst.ComponentName(); compName { case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: version := m.bindVersion(compName, base.Version) tb.Download(compName, inst.OS(), inst.Arch(), version). CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) } - hasImported = true + deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Deploy instance %s -> %s", inst.ComponentName(), inst.ID()))) } + }) - // add the instance to ignore list if it marks itself as ignore_exporter - if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) - } + // init scale out config + scaleOutConfigTasks := buildScaleConfigTasks(m, name, topo, newPart, base, gOpt, p) - // Refresh all configuration - t := tb.InitConfig(name, - base.Version, - m.specManager, - inst, - base.User, - true, // always ignore config check result in scale out - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - Cache: specManager.Path(name, spec.TempConfigPath), - }, - ).Build() - refreshConfigTasks = append(refreshConfigTasks, t) - }) + certificateTasks, err := buildCertificateTasks(m, name, newPart, base, gOpt, p) + if err != nil { + return nil, err + } + // always ignore config check result in scale out + gOpt.IgnoreConfigCheck = true + refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, mergedTopo, base, gOpt, nil) // handle dir scheme changes if hasImported { if err := spec.HandleImportPathMigration(name); err != nil { @@ -320,6 +253,8 @@ func buildScaleOutTask( } } + _, noAgentHosts := getMonitorHosts(mergedTopo) + // Deploy monitor relevant components to remote dlTasks, dpTasks, err := buildMonitoredDeployTask( m, @@ -335,8 +270,40 @@ func buildScaleOutTask( if err != nil { return nil, err } - downloadCompTasks = append(downloadCompTasks, convertStepDisplaysToTasks(dlTasks)...) - deployCompTasks = append(deployCompTasks, convertStepDisplaysToTasks(dpTasks)...) + + downloadCompTasks = append(downloadCompTasks, dlTasks...) + deployCompTasks = append(deployCompTasks, dpTasks...) + + // monitor config + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, + name, + uninitializedHosts, + noAgentHosts, + *topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + m.logger, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt, + p, + ) + + // monitor tls file + moniterCertificateTasks, err := buildMonitoredCertificateTasks( + m, + name, + uninitializedHosts, + noAgentHosts, + topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt, + p, + ) + if err != nil { + return nil, err + } + certificateTasks = append(certificateTasks, moniterCertificateTasks...) builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) if err != nil { @@ -346,9 +313,12 @@ func buildScaleOutTask( // stage2 just start and init config if !opt.Stage2 { builder. - Parallel(false, downloadCompTasks...). - Parallel(false, envInitTasks...). - Parallel(false, deployCompTasks...) + ParallelStep("+ Download TiDB components", gOpt.Force, downloadCompTasks...). + ParallelStep("+ Initialize target host environments", gOpt.Force, envInitTasks...). + ParallelStep("+ Deploy TiDB instance", gOpt.Force, deployCompTasks...). + ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). + ParallelStep("+ Generate scale-out config", gOpt.Force, scaleOutConfigTasks...). + ParallelStep("+ Init monitor config", gOpt.Force, monitorConfigTasks...) } if afterDeploy != nil { @@ -363,15 +333,15 @@ func buildScaleOutTask( // don't start the new instance if opt.Stage1 { // save scale out file lock - builder.Func("Create Scale-Out File Lock", func(_ context.Context) error { + builder.Func("Create scale-out file lock", func(_ context.Context) error { return m.specManager.NewScaleOutLock(name, newPart) }) } else { - builder.Func("Start Cluster", func(ctx context.Context) error { + builder.Func("Start new instances", func(ctx context.Context) error { return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg) }). - Parallel(false, refreshConfigTasks...). - Parallel(false, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...) + ParallelStep("+ Refresh components conifgs", gOpt.Force, refreshConfigTasks...). + ParallelStep("+ Reload prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...) } // remove scale-out file lock @@ -387,6 +357,47 @@ func buildScaleOutTask( return builder.Build(), nil } +// buildScaleConfigTasks generates certificate for instance and transfers it to the server +func buildScaleConfigTasks( + m *Manager, + name string, + topo spec.Topology, + newPart spec.Topology, + base *spec.BaseMeta, + gOpt operator.Options, + p *tui.SSHConnectionProps) []*task.StepDisplay { + var ( + scaleConfigTasks []*task.StepDisplay // tasks which are used to copy certificate to remote host + ) + + // copy certificate to remote host + newPart.IterInstance(func(inst spec.Instance) { + deployDir := spec.Abs(base.User, inst.DeployDir()) + // data dir would be empty for components which don't need it + dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(base.User, inst.LogDir()) + + t := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + ScaleConfig( + name, + base.Version, + m.specManager, + topo, + inst, + base.User, + meta.DirPaths{ + Deploy: deployDir, + Data: dataDirs, + Log: logDir, + }, + ).BuildAsStep(fmt.Sprintf(" - Generate scale-out config %s -> %s", inst.ComponentName(), inst.ID())) + scaleConfigTasks = append(scaleConfigTasks, t) + }) + + return scaleConfigTasks +} + type hostInfo struct { ssh int // ssh port of host os string // operating system @@ -394,15 +405,6 @@ type hostInfo struct { // vendor string } -// Deprecated -func convertStepDisplaysToTasks(t []*task.StepDisplay) []task.Task { - tasks := make([]task.Task, 0, len(t)) - for _, sd := range t { - tasks = append(tasks, sd) - } - return tasks -} - func buildMonitoredDeployTask( m *Manager, name string, @@ -422,7 +424,6 @@ func buildMonitoredDeployTask( // monitoring agents for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { version := m.bindVersion(comp, version) - for host, info := range uniqueHosts { // skip deploying monitoring agents if the instance is marked so if noAgentHosts.Exist(host) { @@ -456,28 +457,9 @@ func buildMonitoredDeployTask( filepath.Join(deployDir, "conf"), filepath.Join(deployDir, "scripts"), } - if globalOptions.TLSEnabled { - deployDirs = append(deployDirs, filepath.Join(deployDir, "tls")) - } // Deploy component - tb := task.NewBuilder(m.logger). - UserSSH( - host, - info.ssh, - globalOptions.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). + tb := task.NewSimpleUerSSH(m.logger, host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). Mkdir(globalOptions.User, host, deployDirs...). CopyComponent( comp, @@ -487,52 +469,75 @@ func buildMonitoredDeployTask( "", host, deployDir, - ). - MonitoredConfig( - name, - comp, - host, - globalOptions.ResourceControl, - monitoredOptions, - globalOptions.User, - globalOptions.TLSEnabled, - meta.DirPaths{ - Deploy: deployDir, - Data: []string{dataDir}, - Log: logDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }, ) + deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Deploy %s -> %s", comp, host))) + } + } + return +} - if globalOptions.TLSEnabled && comp == spec.ComponentBlackboxExporter { - ca, innerr := crypto.ReadCA( - name, - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), - m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), - ) - if innerr != nil { - err = innerr - return +// buildMonitoredCertificateTasks generates certificate for instance and transfers it to the server +func buildMonitoredCertificateTasks( + m *Manager, + name string, + uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch + noAgentHosts set.StringSet, // hosts that do not deploy monitor agents + globalOptions *spec.GlobalOptions, + monitoredOptions *spec.MonitoredOptions, + gOpt operator.Options, + p *tui.SSHConnectionProps, +) ([]*task.StepDisplay, error) { + var certificateTasks []*task.StepDisplay + + if monitoredOptions == nil { + return certificateTasks, nil + } + + if globalOptions.TLSEnabled { + // monitoring agents + for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { + for host, info := range uniqueHosts { + // skip deploying monitoring agents if the instance is marked so + if noAgentHosts.Exist(host) { + continue } - tb = tb.TLSCert( - host, - spec.ComponentBlackboxExporter, - spec.ComponentBlackboxExporter, - monitoredOptions.BlackboxExporterPort, - ca, - meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) - } - deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", comp, host))) + deployDir := spec.Abs(globalOptions.User, monitoredOptions.DeployDir) + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + + // Deploy component + tb := task.NewSimpleUerSSH(m.logger, host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). + Mkdir(globalOptions.User, host, tlsDir) + + if comp == spec.ComponentBlackboxExporter { + ca, innerr := crypto.ReadCA( + name, + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), + ) + if innerr != nil { + return certificateTasks, innerr + } + tb = tb.TLSCert( + host, + spec.ComponentBlackboxExporter, + spec.ComponentBlackboxExporter, + monitoredOptions.BlackboxExporterPort, + ca, + meta.DirPaths{ + Deploy: deployDir, + Cache: m.specManager.Path(name, spec.TempConfigPath), + }) + } + + certificateTasks = append(certificateTasks, tb.BuildAsStep(fmt.Sprintf(" - Generate certificate %s -> %s", comp, host))) + } } } - return + return certificateTasks, nil } -func buildRefreshMonitoredConfigTasks( +func buildInitMonitoredConfigTasks( specManager *spec.SpecManager, name string, uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch @@ -566,23 +571,8 @@ func buildRefreshMonitoredConfigTasks( // log dir will always be with values, but might not used by the component logDir := spec.Abs(globalOptions.User, monitoredOptions.LogDir) // Generate configs - t := task.NewBuilder(logger). - UserSSH( - host, - info.ssh, - globalOptions.User, - sshTimeout, - exeTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - p.Password, - p.IdentityFile, - p.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). + + t := task.NewSimpleUerSSH(logger, host, info.ssh, globalOptions.User, gOpt, p, globalOptions.SSHType). MonitoredConfig( name, comp, @@ -598,21 +588,20 @@ func buildRefreshMonitoredConfigTasks( Cache: specManager.Path(name, spec.TempConfigPath), }, ). - BuildAsStep(fmt.Sprintf(" - Refresh config %s -> %s", comp, host)) + BuildAsStep(fmt.Sprintf(" - Generate config %s -> %s", comp, host)) tasks = append(tasks, t) } } return tasks } -func buildRegenConfigTasks( +func buildInitConfigTasks( m *Manager, name string, topo spec.Topology, base *spec.BaseMeta, gOpt operator.Options, nodes []string, - ignoreCheck bool, ) ([]*task.StepDisplay, bool) { var tasks []*task.StepDisplay hasImported := false @@ -656,7 +645,7 @@ func buildRegenConfigTasks( m.specManager, instance, base.User, - ignoreCheck, + gOpt.IgnoreConfigCheck, meta.DirPaths{ Deploy: deployDir, Data: dataDirs, @@ -664,7 +653,7 @@ func buildRegenConfigTasks( Cache: m.specManager.Path(name, spec.TempConfigPath), }, ). - BuildAsStep(fmt.Sprintf(" - Regenerate config %s -> %s", compName, instance.ID())) + BuildAsStep(fmt.Sprintf(" - Generate config %s -> %s", compName, instance.ID())) tasks = append(tasks, t) }) @@ -713,3 +702,154 @@ func buildDownloadSparkTask(inst spec.Instance, logger *logprinter.Logger, gOpt BuildAsStep(fmt.Sprintf(" - Download %s: (%s/%s)", spec.ComponentSpark, inst.OS(), inst.Arch())) } + +// buildTLSTask create enable/disable tls task +func buildTLSTask( + m *Manager, + name string, + metadata spec.Metadata, + gOpt operator.Options, + reloadCertificate bool, + p *tui.SSHConnectionProps, + delFileMap map[string]set.StringSet, +) (task.Task, error) { + topo := metadata.GetTopology() + base := metadata.GetBaseMeta() + + // load certificate file + if topo.BaseTopo().GlobalOptions.TLSEnabled { + tlsDir := m.specManager.Path(name, spec.TLSCertKeyDir) + m.logger.Infof("Generate certificate: %s", color.YellowString(tlsDir)) + if err := m.loadCertificate(name, topo.BaseTopo().GlobalOptions, reloadCertificate); err != nil { + return nil, err + } + } + + certificateTasks, err := buildCertificateTasks(m, name, topo, base, gOpt, p) + if err != nil { + return nil, err + } + + refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, topo, base, gOpt, nil) + + // handle dir scheme changes + if hasImported { + if err := spec.HandleImportPathMigration(name); err != nil { + return task.NewBuilder(m.logger).Build(), err + } + } + + // monitor + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + moniterCertificateTasks, err := buildMonitoredCertificateTasks( + m, + name, + uniqueHosts, + noAgentHosts, + topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt, + p, + ) + if err != nil { + return nil, err + } + + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, + name, + uniqueHosts, + noAgentHosts, + *topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + m.logger, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt, + p, + ) + + builder, err := m.sshTaskBuilder(name, topo, base.User, gOpt) + if err != nil { + return nil, err + } + + builder. + ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). + ParallelStep("+ Copy monitor certificate to remote host", gOpt.Force, moniterCertificateTasks...). + ParallelStep("+ Refresh instance configs", gOpt.Force, refreshConfigTasks...). + ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...). + Func("Save meta", func(_ context.Context) error { + return m.specManager.SaveMeta(name, metadata) + }) + + // cleanup tls files only in tls disable + if !topo.BaseTopo().GlobalOptions.TLSEnabled { + builder.Func("Cleanup TLS files", func(ctx context.Context) error { + return operator.CleanupComponent(ctx, delFileMap) + }) + } + + tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) + if err != nil { + return nil, err + } + + builder. + Func("Restart Cluster", func(ctx context.Context) error { + return operator.Restart(ctx, topo, gOpt, tlsCfg) + }). + Func("Reload PD Members", func(ctx context.Context) error { + return operator.SetPDMember(ctx, name, topo.BaseTopo().GlobalOptions.TLSEnabled, tlsCfg, metadata) + }) + + return builder.Build(), nil +} + +// buildCertificateTasks generates certificate for instance and transfers it to the server +func buildCertificateTasks( + m *Manager, + name string, + topo spec.Topology, + base *spec.BaseMeta, + gOpt operator.Options, + p *tui.SSHConnectionProps) ([]*task.StepDisplay, error) { + var ( + iterErr error + certificateTasks []*task.StepDisplay // tasks which are used to copy certificate to remote host + ) + + if topo.BaseTopo().GlobalOptions.TLSEnabled { + // copy certificate to remote host + topo.IterInstance(func(inst spec.Instance) { + deployDir := spec.Abs(base.User, inst.DeployDir()) + tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) + + tb := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + Mkdir(base.User, inst.GetHost(), deployDir, tlsDir) + + ca, err := crypto.ReadCA( + name, + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), + ) + if err != nil { + iterErr = err + return + } + t := tb.TLSCert( + inst.GetHost(), + inst.ComponentName(), + inst.Role(), + inst.GetMainPort(), + ca, + meta.DirPaths{ + Deploy: deployDir, + Cache: m.specManager.Path(name, spec.TempConfigPath), + }). + BuildAsStep(fmt.Sprintf(" - Generate certificate %s -> %s", inst.ComponentName(), inst.ID())) + certificateTasks = append(certificateTasks, t) + }) + } + return certificateTasks, iterErr +} diff --git a/pkg/cluster/manager/cacert.go b/pkg/cluster/manager/cacert.go index 6d623b411e..48cdfe751a 100644 --- a/pkg/cluster/manager/cacert.go +++ b/pkg/cluster/manager/cacert.go @@ -101,3 +101,60 @@ func genAndSaveClientCert(ca *crypto.CertificateAuthority, name, tlsPath string) return nil } + +// genAndSaveCertificate generate CA and client cert for TLS enabled cluster +func (m *Manager) genAndSaveCertificate(clusterName string, globalOptions *spec.GlobalOptions) (*crypto.CertificateAuthority, error) { + var ca *crypto.CertificateAuthority + if globalOptions.TLSEnabled { + // generate CA + tlsPath := m.specManager.Path(clusterName, spec.TLSCertKeyDir) + if err := utils.CreateDir(tlsPath); err != nil { + return nil, err + } + ca, err := genAndSaveClusterCA(clusterName, tlsPath) + if err != nil { + return nil, err + } + + // generate client cert + if err = genAndSaveClientCert(ca, clusterName, tlsPath); err != nil { + return nil, err + } + } + + return ca, nil +} + +// checkCertificate check if the certificate file exists +// no need to determine whether to enable tls +func (m *Manager) checkCertificate(clusterName string) error { + tlsFiles := []string{ + m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSCACert), + m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSClientKey), + m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.TLSClientCert), + } + + // check if the file exists + for _, file := range tlsFiles { + if !utils.IsExist(file) { + return perrs.Errorf("TLS file: %s does not exist", file) + } + } + return nil +} + +// loadCertificate +// certificate file exists and reload is true +// will reload certificate file +func (m *Manager) loadCertificate(clusterName string, globalOptions *spec.GlobalOptions, reload bool) error { + err := m.checkCertificate(clusterName) + + // no need to reload and the file already exists + if !reload && err == nil { + return nil + } + + _, err = m.genAndSaveCertificate(clusterName, globalOptions) + + return err +} diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index 23b71cbfbf..ae077fc24b 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -55,7 +55,6 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper if err != nil { return err } - // calculate file paths to be deleted before the prompt delFileMap := getCleanupFiles(topo, cleanOpt.CleanupData, cleanOpt.CleanupLog, false, cleanOpt.CleanupAuditLog, cleanOpt.RetainDataRoles, cleanOpt.RetainDataNodes) @@ -153,6 +152,7 @@ type cleanupFiles struct { cleanupAuditLog bool // whether to clean up the tidb server audit log retainDataRoles []string // roles that don't clean up retainDataNodes []string // roles that don't clean up + ansibleImport bool // cluster is ansible deploy delFileMap map[string]set.StringSet } @@ -236,6 +236,13 @@ func (c *cleanupFiles) instanceCleanupFiles(topo spec.Topology) { deployDir := spec.Abs(topo.BaseTopo().GlobalOptions.User, ins.DeployDir()) tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) tlsPath.Insert(tlsDir) + + // ansible deploy + if ins.IsImported() { + ansibleTLSDir := filepath.Join(deployDir, spec.TLSCertKeyDirWithAnsible) + tlsPath.Insert(ansibleTLSDir) + c.ansibleImport = true + } } if c.delFileMap[ins.GetHost()] == nil { @@ -293,6 +300,11 @@ func (c *cleanupFiles) monitorCleanupFiles(topo spec.Topology) { if c.cleanupTLS && !topo.BaseTopo().GlobalOptions.TLSEnabled { tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) tlsPath.Insert(tlsDir) + // ansible deploy + if c.ansibleImport { + ansibleTLSDir := filepath.Join(deployDir, spec.TLSCertKeyDirWithAnsible) + tlsPath.Insert(ansibleTLSDir) + } } if c.delFileMap[host] == nil { diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index b8fdf84518..bf63ad08ea 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -30,9 +30,8 @@ import ( operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/environment" - "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" @@ -185,90 +184,74 @@ func (m *Manager) Deploy( ) // Initialize environment - uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch - noAgentHosts := set.NewStringSet() - globalOptions := base.GlobalOptions - // generate CA and client cert for TLS enabled cluster - var ca *crypto.CertificateAuthority - if globalOptions.TLSEnabled { - // generate CA - tlsPath := m.specManager.Path(name, spec.TLSCertKeyDir) - if err := utils.CreateDir(tlsPath); err != nil { - return err - } - ca, err = genAndSaveClusterCA(name, tlsPath) - if err != nil { - return err - } + globalOptions := base.GlobalOptions - // generate client cert - if err = genAndSaveClientCert(ca, name, tlsPath); err != nil { - return err - } - } + metadata.SetUser(globalOptions.User) + metadata.SetVersion(clusterVersion) var iterErr error // error when itering over instances iterErr = nil + topo.IterInstance(func(inst spec.Instance) { - if _, found := uniqueHosts[inst.GetHost()]; !found { - // check for "imported" parameter, it can not be true when deploying and scaling out - // only for tidb now, need to support dm - if inst.IsImported() && m.sysName == "tidb" { - iterErr = errors.New( - "'imported' is set to 'true' for new instance, this is only used " + - "for instances imported from tidb-ansible and make no sense when " + - "deploying new instances, please delete the line or set it to 'false' for new instances") - return // skip the host to avoid issues - } + // check for "imported" parameter, it can not be true when deploying and scaling out + // only for tidb now, need to support dm + if inst.IsImported() && m.sysName == "tidb" { + iterErr = errors.New( + "'imported' is set to 'true' for new instance, this is only used " + + "for instances imported from tidb-ansible and make no sense when " + + "deploying new instances, please delete the line or set it to 'false' for new instances") + return // skip the host to avoid issues + } + }) - // add the instance to ignore list if it marks itself as ignore_exporter - if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) - } + // generate CA and client cert for TLS enabled cluster + _, err = m.genAndSaveCertificate(name, globalOptions) + if err != nil { + return err + } - uniqueHosts[inst.GetHost()] = hostInfo{ - ssh: inst.GetSSHPort(), - os: inst.OS(), - arch: inst.Arch(), - } - var dirs []string - for _, dir := range []string{globalOptions.DeployDir, globalOptions.LogDir} { - if dir == "" { - continue - } - dirs = append(dirs, spec.Abs(globalOptions.User, dir)) - } - // the default, relative path of data dir is under deploy dir - if strings.HasPrefix(globalOptions.DataDir, "/") { - dirs = append(dirs, globalOptions.DataDir) + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + + for host, hostInfo := range uniqueHosts { + var dirs []string + for _, dir := range []string{globalOptions.DeployDir, globalOptions.LogDir} { + if dir == "" { + continue } - t := task.NewBuilder(m.logger). - RootSSH( - inst.GetHost(), - inst.GetSSHPort(), - opt.User, - sshConnProps.Password, - sshConnProps.IdentityFile, - sshConnProps.IdentityFilePassphrase, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - sshProxyProps.Password, - sshProxyProps.IdentityFile, - sshProxyProps.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). - EnvInit(inst.GetHost(), globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). - Mkdir(globalOptions.User, inst.GetHost(), dirs...). - BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", inst.GetHost(), inst.GetSSHPort())) - envInitTasks = append(envInitTasks, t) + + dirs = append(dirs, spec.Abs(globalOptions.User, dir)) + } + // the default, relative path of data dir is under deploy dir + if strings.HasPrefix(globalOptions.DataDir, "/") { + dirs = append(dirs, globalOptions.DataDir) } - }) + + t := task.NewBuilder(m.logger). + RootSSH( + host, + hostInfo.ssh, + opt.User, + sshConnProps.Password, + sshConnProps.IdentityFile, + sshConnProps.IdentityFilePassphrase, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + sshProxyProps.Password, + sshProxyProps.IdentityFile, + sshProxyProps.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + globalOptions.SSHType, + ). + EnvInit(host, globalOptions.User, globalOptions.Group, opt.SkipCreateUser || globalOptions.User == opt.User). + Mkdir(globalOptions.User, host, dirs...). + BuildAsStep(fmt.Sprintf(" - Prepare %s:%d", host, hostInfo.ssh)) + envInitTasks = append(envInitTasks, t) + } if iterErr != nil { return iterErr @@ -293,26 +276,8 @@ func (m *Manager) Deploy( filepath.Join(deployDir, "conf"), filepath.Join(deployDir, "scripts"), } - if globalOptions.TLSEnabled { - deployDirs = append(deployDirs, filepath.Join(deployDir, "tls")) - } - t := task.NewBuilder(m.logger). - UserSSH( - inst.GetHost(), - inst.GetSSHPort(), - globalOptions.User, - gOpt.SSHTimeout, - gOpt.OptTimeout, - gOpt.SSHProxyHost, - gOpt.SSHProxyPort, - gOpt.SSHProxyUser, - sshProxyProps.Password, - sshProxyProps.IdentityFile, - sshProxyProps.IdentityFilePassphrase, - gOpt.SSHProxyTimeout, - gOpt.SSHType, - globalOptions.SSHType, - ). + + t := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), globalOptions.User, gOpt, sshProxyProps, globalOptions.SSHType). Mkdir(globalOptions.User, inst.GetHost(), deployDirs...). Mkdir(globalOptions.User, inst.GetHost(), dataDirs...) @@ -344,36 +309,6 @@ func (m *Manager) Deploy( } } - // generate and transfer tls cert for instance - if globalOptions.TLSEnabled { - t = t.TLSCert( - inst.GetHost(), - inst.ComponentName(), - inst.Role(), - inst.GetMainPort(), - ca, - meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) - } - - // generate configs for the component - t = t.InitConfig( - name, - clusterVersion, - m.specManager, - inst, - globalOptions.User, - opt.IgnoreConfigCheck, - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }, - ) - deployCompTasks = append(deployCompTasks, t.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", inst.ComponentName(), inst.GetHost())), ) @@ -383,6 +318,14 @@ func (m *Manager) Deploy( return iterErr } + // generates certificate for instance and transfers it to the server + certificateTasks, err := buildCertificateTasks(m, name, topo, metadata.GetBaseMeta(), gOpt, sshProxyProps) + if err != nil { + return err + } + + refreshConfigTasks, _ := buildInitConfigTasks(m, name, topo, metadata.GetBaseMeta(), gOpt, nil) + // Deploy monitor relevant components to remote dlTasks, dpTasks, err := buildMonitoredDeployTask( m, @@ -401,6 +344,35 @@ func (m *Manager) Deploy( downloadCompTasks = append(downloadCompTasks, dlTasks...) deployCompTasks = append(deployCompTasks, dpTasks...) + // monitor tls file + moniterCertificateTasks, err := buildMonitoredCertificateTasks( + m, + name, + uniqueHosts, + noAgentHosts, + topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + gOpt, + sshProxyProps, + ) + if err != nil { + return err + } + certificateTasks = append(certificateTasks, moniterCertificateTasks...) + + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, + name, + uniqueHosts, + noAgentHosts, + *topo.BaseTopo().GlobalOptions, + topo.GetMonitoredOptions(), + m.logger, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt, + sshProxyProps, + ) builder := task.NewBuilder(m.logger). Step("+ Generate SSH keys", task.NewBuilder(m.logger). @@ -409,7 +381,10 @@ func (m *Manager) Deploy( m.logger). ParallelStep("+ Download TiDB components", false, downloadCompTasks...). ParallelStep("+ Initialize target host environments", false, envInitTasks...). - ParallelStep("+ Copy files", false, deployCompTasks...) + ParallelStep("+ Deploy TiDB instance", false, deployCompTasks...). + ParallelStep("+ Copy certificate to remote host", gOpt.Force, certificateTasks...). + ParallelStep("+ Init instance configs", gOpt.Force, refreshConfigTasks...). + ParallelStep("+ Init monitor configs", gOpt.Force, monitorConfigTasks...) if afterDeploy != nil { afterDeploy(builder, topo, gOpt) @@ -430,8 +405,6 @@ func (m *Manager) Deploy( return err } - metadata.SetUser(globalOptions.User) - metadata.SetVersion(clusterVersion) err = m.specManager.SaveMeta(name, metadata) if err != nil { diff --git a/pkg/cluster/manager/destroy.go b/pkg/cluster/manager/destroy.go index 9413b17e99..00fd5db62b 100644 --- a/pkg/cluster/manager/destroy.go +++ b/pkg/cluster/manager/destroy.go @@ -139,7 +139,11 @@ func (m *Manager) DestroyTombstone( if err != nil { return err } - regenConfigTasks, _ := buildRegenConfigTasks(m, name, topo, base, gOpt, nodes, true) + + // Destroy ignore error and force exec + gOpt.IgnoreConfigCheck = true + gOpt.Force = true + regenConfigTasks, _ := buildInitConfigTasks(m, name, topo, base, gOpt, nodes) t := b. Func("FindTomestoneNodes", func(ctx context.Context) (err error) { @@ -157,8 +161,8 @@ func (m *Manager) DestroyTombstone( ClusterOperate(cluster, operator.DestroyTombstoneOperation, gOpt, tlsCfg). UpdateMeta(name, clusterMeta, nodes). UpdateTopology(name, m.specManager.Path(name), clusterMeta, nodes). - ParallelStep("+ Refresh instance configs", true, regenConfigTasks...). - Parallel(true, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...). + ParallelStep("+ Refresh instance configs", gOpt.Force, regenConfigTasks...). + ParallelStep("+ Reloda prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...). Build() if err := t.Execute(ctx); err != nil { diff --git a/pkg/cluster/manager/reload.go b/pkg/cluster/manager/reload.go index fc0fcea2cf..e75fcc976d 100644 --- a/pkg/cluster/manager/reload.go +++ b/pkg/cluster/manager/reload.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/set" + "github.com/pingcap/tiup/pkg/tui" ) @@ -73,25 +73,20 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo topo := metadata.GetTopology() base := metadata.GetBaseMeta() - uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch - noAgentHosts := set.NewStringSet() - topo.IterInstance(func(inst spec.Instance) { - // add the instance to ignore list if it marks itself as ignore_exporter - if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) - } + // monitor + uniqueHosts, noAgentHosts := getMonitorHosts(topo) - if _, found := uniqueHosts[inst.GetHost()]; !found { - uniqueHosts[inst.GetHost()] = hostInfo{ - ssh: inst.GetSSHPort(), - os: inst.OS(), - arch: inst.Arch(), - } + // init config + refreshConfigTasks, hasImported := buildInitConfigTasks(m, name, topo, base, gOpt, nil) + + // handle dir scheme changes + if hasImported { + if err := spec.HandleImportPathMigration(name); err != nil { + return err } - }) + } - refreshConfigTasks, hasImported := buildRegenConfigTasks(m, name, topo, base, gOpt, nil, gOpt.IgnoreConfigCheck) - monitorConfigTasks := buildRefreshMonitoredConfigTasks( + monitorConfigTasks := buildInitMonitoredConfigTasks( m.specManager, name, uniqueHosts, @@ -105,13 +100,6 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo sshProxyProps, ) - // handle dir scheme changes - if hasImported { - if err := spec.HandleImportPathMigration(name); err != nil { - return err - } - } - b, err := m.sshTaskBuilder(name, topo, base.User, gOpt) if err != nil { return err @@ -135,7 +123,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo if err != nil { return err } - b.Func("UpgradeCluster", func(ctx context.Context) error { + b.Func("Upgrade Cluster", func(ctx context.Context) error { return operator.Upgrade(ctx, topo, gOpt, tlsCfg) }) } diff --git a/pkg/cluster/manager/scale_in.go b/pkg/cluster/manager/scale_in.go index f03d253a0b..0370e299fe 100644 --- a/pkg/cluster/manager/scale_in.go +++ b/pkg/cluster/manager/scale_in.go @@ -92,7 +92,8 @@ func (m *Manager) ScaleIn( base := metadata.GetBaseMeta() // Regenerate configuration - regenConfigTasks, hasImported := buildRegenConfigTasks(m, name, topo, base, gOpt, nodes, true) + gOpt.IgnoreConfigCheck = true + regenConfigTasks, hasImported := buildInitConfigTasks(m, name, topo, base, gOpt, nodes) // handle dir scheme changes if hasImported { @@ -115,7 +116,7 @@ func (m *Manager) ScaleIn( t := b. ParallelStep("+ Refresh instance configs", force, regenConfigTasks...). - Parallel(force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...). + ParallelStep("+ Reloda prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...). Build() ctx := ctxt.New( diff --git a/pkg/cluster/manager/tls.go b/pkg/cluster/manager/tls.go new file mode 100644 index 0000000000..5e6eab2b14 --- /dev/null +++ b/pkg/cluster/manager/tls.go @@ -0,0 +1,182 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + "fmt" + "os" + + "github.com/fatih/color" + "github.com/joomcode/errorx" + perrs "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/cluster/clusterutil" + "github.com/pingcap/tiup/pkg/cluster/ctxt" + "github.com/pingcap/tiup/pkg/cluster/executor" + operator "github.com/pingcap/tiup/pkg/cluster/operation" + "github.com/pingcap/tiup/pkg/cluster/spec" + + "github.com/pingcap/tiup/pkg/set" + "github.com/pingcap/tiup/pkg/tui" + "golang.org/x/mod/semver" +) + +// TLS set cluster enable/disable encrypt communication by tls +func (m *Manager) TLS(name string, gOpt operator.Options, enable, cleanCertificate, reloadCertificate, skipConfirm bool) error { + if err := clusterutil.ValidateClusterNameOrError(name); err != nil { + return err + } + + // check locked + if err := m.specManager.ScaleOutLockedErr(name); err != nil { + return err + } + + metadata, err := m.meta(name) + if err != nil { + return err + } + + topo := metadata.GetTopology() + base := metadata.GetBaseMeta() + + // set tls_enabled + globalOptions := topo.BaseTopo().GlobalOptions + // if force is true, skip this check + if globalOptions.TLSEnabled == enable && !gOpt.Force { + if enable { + m.logger.Infof("cluster `%s` TLS status is already enable\n", name) + } else { + m.logger.Infof("cluster `%s` TLS status is already disable\n", name) + } + return nil + } + globalOptions.TLSEnabled = enable + + if err := checkTLSEnv(topo, name, base.Version, skipConfirm); err != nil { + return err + } + + var ( + sshProxyProps *tui.SSHConnectionProps = &tui.SSHConnectionProps{} + ) + if gOpt.SSHType != executor.SSHTypeNone { + var err error + if len(gOpt.SSHProxyHost) != 0 { + if sshProxyProps, err = tui.ReadIdentityFileOrPassword(gOpt.SSHProxyIdentity, gOpt.SSHProxyUsePassword); err != nil { + return err + } + } + } + + // delFileMap: files that need to be cleaned up, if flag -- cleanCertificate are used + delFileMap, err := getTLSFileMap(m, name, topo, enable, cleanCertificate, skipConfirm) + if err != nil { + return err + } + + // Build the tls tasks + t, err := buildTLSTask( + m, name, metadata, gOpt, reloadCertificate, sshProxyProps, delFileMap) + if err != nil { + return err + } + + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := t.Execute(ctx); err != nil { + if errorx.Cast(err) != nil { + // FIXME: Map possible task errors and give suggestions. + return err + } + return perrs.Trace(err) + } + + if err := m.specManager.SaveMeta(name, metadata); err != nil { + return err + } + + if !enable { + // the cleanCertificate parameter will only take effect when enable is false + if cleanCertificate { + os.RemoveAll(m.specManager.Path(name, spec.TLSCertKeyDir)) + } + m.logger.Infof("\tCleanup localhost tls file success") + } + + if enable { + m.logger.Infof("Enable cluster `%s` TLS between TiDB components successfully", name) + } else { + m.logger.Infof("Disable cluster `%s` TLS between TiDB components successfully", name) + } + return nil +} + +// checkTLSEnv check tiflash vserson and show confirm +func checkTLSEnv(topo spec.Topology, clusterName, version string, skipConfirm bool) error { + // check tiflash version + if clusterSpec, ok := topo.(*spec.Specification); ok { + if clusterSpec.GlobalOptions.TLSEnabled { + if semver.Compare(version, "v4.0.5") < 0 && len(clusterSpec.TiFlashServers) > 0 { + return fmt.Errorf("TiFlash %s is not supported in TLS enabled cluster", version) + } + } + + if len(clusterSpec.PDServers) != 1 { + return errorx.EnsureStackTrace(fmt.Errorf("Multiple PD nodes is not supported enable/disable TLS")). + WithProperty(tui.SuggestionFromString("Please `scale-in` PD nodes to one and try again.")) + } + } + + if !skipConfirm { + return tui.PromptForConfirmOrAbortError( + fmt.Sprintf("Enable/Disable TLS will %s the cluster `%s`\nDo you want to continue? [y/N]:", + color.HiYellowString("restart"), + color.HiYellowString(clusterName), + )) + } + return nil +} + +// getTLSFileMap +func getTLSFileMap(m *Manager, clusterName string, topo spec.Topology, + enableTLS, cleanCertificate, skipConfirm bool) (map[string]set.StringSet, error) { + delFileMap := make(map[string]set.StringSet) + + if !enableTLS && cleanCertificate { + // get: host: set(tlsdir) + delFileMap = getCleanupFiles(topo, false, false, cleanCertificate, false, []string{}, []string{}) + // build file list string + delFileList := fmt.Sprintf("\n%s:\n %s", color.CyanString("localhost"), m.specManager.Path(clusterName, spec.TLSCertKeyDir)) + for host, fileList := range delFileMap { + delFileList += fmt.Sprintf("\n%s:", color.CyanString(host)) + for _, dfp := range fileList.Slice() { + delFileList += fmt.Sprintf("\n %s", dfp) + } + } + + m.logger.Warnf("The parameter `%s` will delete the following files: %s", color.YellowString("--clean-certificate"), delFileList) + + if !skipConfirm { + if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { + return delFileMap, err + } + } + } + + return delFileMap, nil +} diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index 2a7e4a50de..a899852e7b 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -337,7 +337,7 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) e := ctxt.GetInner(ctx).Get(ins.GetHost()) - logger.Infof("Destroying instance %s", ins.GetHost()) + logger.Infof("\tDestroying instance %s", ins.GetHost()) var dataDirs []string if len(ins.DataDir()) > 0 { @@ -377,7 +377,7 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T delPaths.Insert(filepath.Join(deployDir, "bin")) delPaths.Insert(filepath.Join(deployDir, "scripts")) if cls.BaseTopo().GlobalOptions.TLSEnabled { - delPaths.Insert(filepath.Join(deployDir, "tls")) + delPaths.Insert(filepath.Join(deployDir, spec.TLSCertKeyDir)) } // only delete path if it is not used by any other instance in the cluster if strings.HasPrefix(logDir, deployDir) && cls.CountDir(ins.GetHost(), logDir) == 1 { diff --git a/pkg/cluster/operation/operation.go b/pkg/cluster/operation/operation.go index 16555455eb..1cd42f839c 100644 --- a/pkg/cluster/operation/operation.go +++ b/pkg/cluster/operation/operation.go @@ -31,7 +31,7 @@ const ( type Options struct { Roles []string Nodes []string - Force bool // Option for upgrade subcommand + Force bool // Option for upgrade/tls subcommand SSHTimeout uint64 // timeout in seconds when connecting an SSH server OptTimeout uint64 // timeout in seconds for operations that support it, not to confuse with SSH timeout APITimeout uint64 // timeout in seconds for API operations that support it, like transferring store leader diff --git a/pkg/cluster/operation/pd_member.go b/pkg/cluster/operation/pd_member.go new file mode 100644 index 0000000000..f0171c7aad --- /dev/null +++ b/pkg/cluster/operation/pd_member.go @@ -0,0 +1,96 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + "context" + "crypto/tls" + "fmt" + "strings" + "time" + + "github.com/pingcap/tiup/pkg/cluster/spec" +) + +// UpdatePDMember is used to update pd cluster member +type UpdatePDMember struct { + cluster string + tlsCfg *tls.Config + metadata spec.Metadata + enableTLS bool +} + +// SetPDMember set the member of pd-etcd +func SetPDMember(ctx context.Context, clusterName string, enable bool, tlsCfg *tls.Config, meta spec.Metadata) error { + u := &UpdatePDMember{ + cluster: clusterName, + tlsCfg: tlsCfg, + metadata: meta, + enableTLS: enable, + } + + return u.Execute(ctx) +} + +// Execute implements the Task interface +func (u *UpdatePDMember) Execute(ctx context.Context) error { + // connection etcd + etcdClient, err := u.metadata.GetTopology().(*spec.Specification).GetEtcdClient(u.tlsCfg) + if err != nil { + return err + } + // etcd client defaults to wait forever + // if all pd were down, don't hang forever + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + // txn := etcdClient.Txn(ctx) + + memberList, err := etcdClient.MemberList(ctx) + if err != nil { + return err + } + + for _, member := range memberList.Members { + _, err := etcdClient.MemberUpdate(ctx, member.GetID(), u.updatePeerURLs(member.PeerURLs)) + if err != nil { + return err + } + } + + // get member list after update + memberList, err = etcdClient.MemberList(ctx) + if err != nil { + return err + } + for _, member := range memberList.Members { + fmt.Printf("\tUpdate %s peerURLs: %v\n", member.Name, member.PeerURLs) + } + + return nil +} + +// updatePeerURLs http->https or https->http +func (u *UpdatePDMember) updatePeerURLs(peerURLs []string) []string { + newPeerURLs := []string{} + + for _, url := range peerURLs { + if u.enableTLS { + newPeerURLs = append(newPeerURLs, strings.Replace(url, "http://", "https://", 1)) + } else { + newPeerURLs = append(newPeerURLs, strings.Replace(url, "https://", "http://", 1)) + } + } + + return newPeerURLs +} diff --git a/pkg/cluster/spec/alertmanager.go b/pkg/cluster/spec/alertmanager.go index 14c5dc528e..722d91952d 100644 --- a/pkg/cluster/spec/alertmanager.go +++ b/pkg/cluster/spec/alertmanager.go @@ -149,6 +149,11 @@ func (i *AlertManagerInstance) InitConfig( WithWebPort(spec.WebPort).WithClusterPort(spec.ClusterPort).WithNumaNode(spec.NumaNode). AppendEndpoints(AlertManagerEndpoints(alertmanagers, deployUser, enableTLS)) + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + fp := filepath.Join(paths.Cache, fmt.Sprintf("run_alertmanager_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { return err @@ -192,3 +197,8 @@ func (i *AlertManagerInstance) ScaleConfig( i.topo = topo return i.InitConfig(ctx, e, clusterName, clusterVersion, deployUser, paths) } + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *AlertManagerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} diff --git a/pkg/cluster/spec/cdc.go b/pkg/cluster/spec/cdc.go index 5f5018e0df..7a53b11921 100644 --- a/pkg/cluster/spec/cdc.go +++ b/pkg/cluster/spec/cdc.go @@ -178,6 +178,10 @@ func (i *CDCInstance) InitConfig( spec.TZ, ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).AppendEndpoints(topo.Endpoints(deployUser)...) + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } if len(paths.Data) != 0 { cfg = cfg.PatchByVersion(clusterVersion, paths.Data[0]) } @@ -198,3 +202,8 @@ func (i *CDCInstance) InitConfig( return i.MergeServerConfig(ctx, e, globalConfig, instanceConfig, paths) } + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *CDCInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} diff --git a/pkg/cluster/spec/drainer.go b/pkg/cluster/spec/drainer.go index 2f374cd696..6066648108 100644 --- a/pkg/cluster/spec/drainer.go +++ b/pkg/cluster/spec/drainer.go @@ -153,7 +153,6 @@ func (i *DrainerInstance) InitConfig( if err := i.BaseInstance.InitConfig(ctx, e, topo.GlobalOptions, deployUser, paths); err != nil { return err } - enableTLS := topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*DrainerSpec) nodeID := i.GetHost() + ":" + strconv.Itoa(i.GetPort()) @@ -181,7 +180,8 @@ func (i *DrainerInstance) InitConfig( return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -209,28 +209,51 @@ func (i *DrainerInstance) InitConfig( } // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *DrainerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ssl-ca"] = fmt.Sprintf( + configs["security.ssl-ca"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.ssl-cert"] = fmt.Sprintf( + configs["security.ssl-cert"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.ssl-key"] = fmt.Sprintf( + configs["security.ssl-key"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ssl-ca", + "security.ssl-cert", + "security.ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } diff --git a/pkg/cluster/spec/grafana.go b/pkg/cluster/spec/grafana.go index 6afa2b6335..285ddc44a7 100644 --- a/pkg/cluster/spec/grafana.go +++ b/pkg/cluster/spec/grafana.go @@ -175,6 +175,12 @@ func (i *GrafanaInstance) InitConfig( ConfigToFile(fp); err != nil { return err } + + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + dst = filepath.Join(paths.Deploy, "conf", "grafana.ini") if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err @@ -222,6 +228,11 @@ func (i *GrafanaInstance) InitConfig( return i.TransferLocalConfigFile(ctx, e, fp, dst) } +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *GrafanaInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + func (i *GrafanaInstance) initDashboards(ctx context.Context, e ctxt.Executor, spec *GrafanaSpec, paths meta.DirPaths, clusterName string) error { dashboardsDir := filepath.Join(paths.Deploy, "dashboards") if spec.DashboardDir != "" { diff --git a/pkg/cluster/spec/instance.go b/pkg/cluster/spec/instance.go index 0b92cae164..0bcd46b387 100644 --- a/pkg/cluster/spec/instance.go +++ b/pkg/cluster/spec/instance.go @@ -102,6 +102,7 @@ type Instance interface { Arch() string IsPatched() bool SetPatched(bool) + setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) } // PortStarted wait until a port is being listened @@ -190,9 +191,20 @@ func (i *BaseInstance) InitConfig(ctx context.Context, e ctxt.Executor, opt Glob return errors.Annotatef(err, "execute: %s", cmd) } + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + return nil } +// setTLSConfig set TLS Config to support enable/disable TLS +// baseInstance no need to configure TLS +func (i *BaseInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // TransferLocalConfigFile scp local config file to remote // Precondition: the user on remote have permission to access & mkdir of dest files func (i *BaseInstance) TransferLocalConfigFile(ctx context.Context, e ctxt.Executor, local, remote string) error { @@ -330,6 +342,11 @@ func (i *BaseInstance) DeployDir() string { return reflect.Indirect(reflect.ValueOf(i.InstanceSpec)).FieldByName("DeployDir").String() } +// TLSDir implements Instance interface +func (i *BaseInstance) TLSDir() string { + return filepath.Join(i.DeployDir()) +} + // DataDir implements Instance interface func (i *BaseInstance) DataDir() string { dataDir := reflect.Indirect(reflect.ValueOf(i.InstanceSpec)).FieldByName("DataDir") diff --git a/pkg/cluster/spec/monitoring.go b/pkg/cluster/spec/monitoring.go index 77771c12f2..dcde45027b 100644 --- a/pkg/cluster/spec/monitoring.go +++ b/pkg/cluster/spec/monitoring.go @@ -297,6 +297,12 @@ func (i *MonitorInstance) InitConfig( if err != nil { return err } + + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + cfig.SetRemoteConfig(string(remoteCfg)) for _, alertmanager := range spec.ExternalAlertmanagers { @@ -355,6 +361,11 @@ func (i *MonitorInstance) InitConfig( return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".yml", paths, nil) } +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *MonitorInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // We only really installRules for dm cluster because the rules(*.rules.yml) packed with the prometheus // component is designed for tidb cluster (the dm cluster use the same prometheus component with tidb // cluster), and the rules for dm cluster is packed in the dm-master component. So if deploying tidb diff --git a/pkg/cluster/spec/pd.go b/pkg/cluster/spec/pd.go index 1f683b4366..fd47a02769 100644 --- a/pkg/cluster/spec/pd.go +++ b/pkg/cluster/spec/pd.go @@ -193,7 +193,8 @@ func (i *PDInstance) InitConfig( if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -220,31 +221,55 @@ func (i *PDInstance) InitConfig( } } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *PDInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { // set TLS configs if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.cacert-path"] = fmt.Sprintf( + configs["security.cacert-path"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cert-path"] = fmt.Sprintf( + configs["security.cert-path"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.key-path"] = fmt.Sprintf( + configs["security.key-path"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.cacert-path", + "security.cert-path", + "security.key-path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } // ScaleConfig deploy temporary config on scaling @@ -264,7 +289,6 @@ func (i *PDInstance) ScaleConfig( } cluster := mustBeClusterTopo(topo) - spec := i.InstanceSpec.(*PDSpec) cfg0 := scripts.NewPDScript( i.Name, diff --git a/pkg/cluster/spec/profile.go b/pkg/cluster/spec/profile.go index 813744f32b..cf852cb2e7 100644 --- a/pkg/cluster/spec/profile.go +++ b/pkg/cluster/spec/profile.go @@ -27,15 +27,16 @@ import ( // sub directory names const ( - TiUPPackageCacheDir = "packages" - TiUPClusterDir = "clusters" - TiUPAuditDir = "audit" - TLSCertKeyDir = "tls" - TLSCACert = "ca.crt" - TLSCAKey = "ca.pem" - TLSClientCert = "client.crt" - TLSClientKey = "client.pem" - PFXClientCert = "client.pfx" + TiUPPackageCacheDir = "packages" + TiUPClusterDir = "clusters" + TiUPAuditDir = "audit" + TLSCertKeyDir = "tls" + TLSCertKeyDirWithAnsible = "ssl" + TLSCACert = "ca.crt" + TLSCAKey = "ca.pem" + TLSClientCert = "client.crt" + TLSClientKey = "client.pem" + PFXClientCert = "client.pfx" ) var profileDir string diff --git a/pkg/cluster/spec/pump.go b/pkg/cluster/spec/pump.go index 696d04f728..ddea53e97c 100644 --- a/pkg/cluster/spec/pump.go +++ b/pkg/cluster/spec/pump.go @@ -176,7 +176,8 @@ func (i *PumpInstance) InitConfig( return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -203,25 +204,49 @@ func (i *PumpInstance) InitConfig( } } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + return i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *PumpInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { // set TLS configs if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ssl-ca"] = fmt.Sprintf( + configs["security.ssl-ca"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.ssl-cert"] = fmt.Sprintf( + configs["security.ssl-cert"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.ssl-key"] = fmt.Sprintf( + configs["security.ssl-key"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ssl-ca", + "security.ssl-cert", + "security.ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - return i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths) + return configs, nil } diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index 6d3e73afae..eb7e653321 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -23,12 +23,14 @@ import ( "time" "github.com/creasty/defaults" + "github.com/joomcode/errorx" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/proxy" + "github.com/pingcap/tiup/pkg/tui" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "golang.org/x/mod/semver" @@ -216,7 +218,13 @@ func (s *Specification) TLSConfig(dir string) (*tls.Config, error) { if !s.GlobalOptions.TLSEnabled { return nil, nil } - return LoadClientCert(dir) + tlsConfig, err := LoadClientCert(dir) + if err != nil { + return nil, errorx.EnsureStackTrace(err). + WithProperty(tui.SuggestionFromString("TLS is enabled, but the TLS configuration cannot be obtained")) + } + + return tlsConfig, nil } // Type implements Topology interface. diff --git a/pkg/cluster/spec/spec_test.go b/pkg/cluster/spec/spec_test.go index 335a9c7b12..ebe915f874 100644 --- a/pkg/cluster/spec/spec_test.go +++ b/pkg/cluster/spec/spec_test.go @@ -15,6 +15,7 @@ package spec import ( "bytes" + "context" "strings" "testing" @@ -585,6 +586,7 @@ pd_servers: } func (s *metaSuiteTopo) TestTiFlashStorageSection(c *C) { + ctx := context.Background() spec := &Specification{} err := yaml.Unmarshal([]byte(` tiflash_servers: @@ -605,7 +607,7 @@ tiflash_servers: // This should be the same with tiflash_server instance's "data_dir" dataDir := "/hdd0/tiflash,/hdd1/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.(*TiFlashInstance).initTiFlashConfig(cfg, "v4.0.8", spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.(*TiFlashInstance).initTiFlashConfig(ctx, cfg, "v4.0.8", spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) path, ok := conf["path"] @@ -617,7 +619,7 @@ tiflash_servers: ins := instances[0].(*TiFlashInstance) dataDir := "/ssd0/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.initTiFlashConfig(ctx, cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) _, ok := conf["path"] @@ -768,6 +770,8 @@ tiflash_servers: `), spec) c.Assert(err, IsNil) + ctx := context.Background() + flashComp := FindComponent(spec, ComponentTiFlash) instances := flashComp.Instances() c.Assert(len(instances), Equals, 1) @@ -777,7 +781,7 @@ tiflash_servers: ins := instances[0].(*TiFlashInstance) dataDir := "/ssd0/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.initTiFlashConfig(ctx, cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) // We need an empty string for 'users.default.password' for backward compatibility. Or the TiFlash process will fail to start with older versions @@ -800,7 +804,7 @@ tiflash_servers: ins := instances[0].(*TiFlashInstance) dataDir := "/ssd0/tiflash" cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "") - conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) + conf, err := ins.initTiFlashConfig(ctx, cfg, ver, spec.ServerConfigs.TiFlash, meta.DirPaths{}) c.Assert(err, IsNil) // Those deprecated settings are ignored in newer versions diff --git a/pkg/cluster/spec/tidb.go b/pkg/cluster/spec/tidb.go index 86c7351eea..399777832e 100644 --- a/pkg/cluster/spec/tidb.go +++ b/pkg/cluster/spec/tidb.go @@ -154,7 +154,8 @@ func (i *TiDBInstance) InitConfig( if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -181,31 +182,55 @@ func (i *TiDBInstance) InitConfig( } } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *TiDBInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { // set TLS configs if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.cluster-ssl-ca"] = fmt.Sprintf( + configs["security.cluster-ssl-ca"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cluster-ssl-cert"] = fmt.Sprintf( + configs["security.cluster-ssl-cert"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.cluster-ssl-key"] = fmt.Sprintf( + configs["security.cluster-ssl-key"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.cluster-ssl-ca", + "security.cluster-ssl-cert", + "security.cluster-ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } // ScaleConfig deploy temporary config on scaling diff --git a/pkg/cluster/spec/tiflash.go b/pkg/cluster/spec/tiflash.go index 7a2081c34a..3fb7b33ac1 100644 --- a/pkg/cluster/spec/tiflash.go +++ b/pkg/cluster/spec/tiflash.go @@ -370,7 +370,7 @@ func checkTiFlashStorageConfigWithVersion(clusterVersion string, config map[stri } // InitTiFlashConfig initializes TiFlash config file with the configurations in server_configs -func (i *TiFlashInstance) initTiFlashConfig(cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { +func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { var ( pathConfig string isStorageDirsDefined bool @@ -416,26 +416,15 @@ func (i *TiFlashInstance) initTiFlashConfig(cfg *scripts.TiFlashScript, clusterV spec := i.InstanceSpec.(*TiFlashSpec) port := "http_port" - // set TLS configs enableTLS := i.topo.(*Specification).GlobalOptions.TLSEnabled + if enableTLS { port = "https_port" - if spec.Config == nil { - spec.Config = make(map[string]interface{}) - } - spec.Config["security.ca_path"] = fmt.Sprintf( - "%s/tls/%s", - paths.Deploy, - TLSCACert, - ) - spec.Config["security.cert_path"] = fmt.Sprintf( - "%s/tls/%s.crt", - paths.Deploy, - i.Role()) - spec.Config["security.key_path"] = fmt.Sprintf( - "%s/tls/%s.pem", - paths.Deploy, - i.Role()) + } + // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return nil, err } topo := Specification{} @@ -504,7 +493,7 @@ func (i *TiFlashInstance) mergeTiFlashInstanceConfig(clusterVersion string, glob } // InitTiFlashLearnerConfig initializes TiFlash learner config file -func (i *TiFlashInstance) InitTiFlashLearnerConfig(cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { +func (i *TiFlashInstance) InitTiFlashLearnerConfig(ctx context.Context, cfg *scripts.TiFlashScript, clusterVersion string, src map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { topo := Specification{} var statusAddr string @@ -541,27 +530,87 @@ server_configs: enableTLS := i.topo.(*Specification).GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*TiFlashSpec) // set TLS configs + spec.Config, err = i.setTLSConfigWithTiFlashLearner(ctx, enableTLS, spec.Config, paths) + if err != nil { + return nil, err + } + + conf := MergeConfig(topo.ServerConfigs.TiFlashLearner, spec.Config, src) + return conf, nil +} + +// setTLSConfigWithTiFlashLearner set TLS Config to support enable/disable TLS +func (i *TiFlashInstance) setTLSConfigWithTiFlashLearner(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ca-path"] = fmt.Sprintf( + configs["security.ca-path"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cert-path"] = fmt.Sprintf( + configs["security.cert-path"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.key-path"] = fmt.Sprintf( + configs["security.key-path"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ca-path", + "security.cert-path", + "security.key-path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - conf := MergeConfig(topo.ServerConfigs.TiFlashLearner, spec.Config, src) - return conf, nil + return configs, nil +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *TiFlashInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + if enableTLS { + if configs == nil { + configs = make(map[string]interface{}) + } + configs["security.ca_path"] = fmt.Sprintf( + "%s/tls/%s", + paths.Deploy, + TLSCACert, + ) + configs["security.cert_path"] = fmt.Sprintf( + "%s/tls/%s.crt", + paths.Deploy, + i.Role()) + configs["security.key_path"] = fmt.Sprintf( + "%s/tls/%s.pem", + paths.Deploy, + i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ca_path", + "security.cert_path", + "security.key_path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } + } + + return configs, nil } // InitConfig implement Instance interface @@ -619,7 +668,7 @@ func (i *TiFlashInstance) InitConfig( return err } - conf, err := i.InitTiFlashLearnerConfig(cfg, clusterVersion, topo.ServerConfigs.TiFlashLearner, paths) + conf, err := i.InitTiFlashLearnerConfig(ctx, cfg, clusterVersion, topo.ServerConfigs.TiFlashLearner, paths) if err != nil { return err } @@ -652,7 +701,7 @@ func (i *TiFlashInstance) InitConfig( } // Init the configuration using cfg and server_configs - if conf, err = i.initTiFlashConfig(cfg, clusterVersion, topo.ServerConfigs.TiFlash, paths); err != nil { + if conf, err = i.initTiFlashConfig(ctx, cfg, clusterVersion, topo.ServerConfigs.TiFlash, paths); err != nil { return err } diff --git a/pkg/cluster/spec/tikv.go b/pkg/cluster/spec/tikv.go index 7a84e0552a..b4af65e067 100644 --- a/pkg/cluster/spec/tikv.go +++ b/pkg/cluster/spec/tikv.go @@ -231,7 +231,8 @@ func (i *TiKVInstance) InitConfig( return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } @@ -259,30 +260,53 @@ func (i *TiKVInstance) InitConfig( } // set TLS configs + spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) + if err != nil { + return err + } + + if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { + return err + } + + return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *TiKVInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { if enableTLS { - if spec.Config == nil { - spec.Config = make(map[string]interface{}) + if configs == nil { + configs = make(map[string]interface{}) } - spec.Config["security.ca-path"] = fmt.Sprintf( + configs["security.ca-path"] = fmt.Sprintf( "%s/tls/%s", paths.Deploy, TLSCACert, ) - spec.Config["security.cert-path"] = fmt.Sprintf( + configs["security.cert-path"] = fmt.Sprintf( "%s/tls/%s.crt", paths.Deploy, i.Role()) - spec.Config["security.key-path"] = fmt.Sprintf( + configs["security.key-path"] = fmt.Sprintf( "%s/tls/%s.pem", paths.Deploy, i.Role()) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.ca-path", + "security.cert-path", + "security.key-path", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } } - if err := i.MergeServerConfig(ctx, e, globalConfig, spec.Config, paths); err != nil { - return err - } - - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return configs, nil } // ScaleConfig deploy temporary config on scaling diff --git a/pkg/cluster/spec/tispark.go b/pkg/cluster/spec/tispark.go index 91afbefffc..eb16fa550d 100644 --- a/pkg/cluster/spec/tispark.go +++ b/pkg/cluster/spec/tispark.go @@ -406,6 +406,12 @@ func (i *TiSparkWorkerInstance) InitConfig( cfg := config.NewTiSparkConfig(pdList).WithMasters(strings.Join(masterList, ",")). WithCustomFields(topo.TiSparkMasters[0].SparkConfigs) + + // doesn't work + if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + return err + } + // transfer spark-defaults.conf fp := filepath.Join(paths.Cache, fmt.Sprintf("spark-defaults-%s-%d.conf", host, port)) if err := cfg.ConfigToFile(fp); err != nil { @@ -460,6 +466,12 @@ func (i *TiSparkWorkerInstance) InitConfig( return e.Transfer(ctx, fp, dst, false, 0, false) } +// setTLSConfig set TLS Config to support enable/disable TLS +// TiSparkWorkerInstance no need to configure TLS +func (i *TiSparkWorkerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { + return nil, nil +} + // ScaleConfig deploy temporary config on scaling func (i *TiSparkWorkerInstance) ScaleConfig( ctx context.Context, diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index 0a8a0386a2..3eb7829d74 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -26,6 +26,7 @@ import ( logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/proxy" + "github.com/pingcap/tiup/pkg/tui" ) // Builder is used to build TiUP task @@ -69,6 +70,27 @@ func (b *Builder) RootSSH( return b } +// NewSimpleUerSSH append a UserSSH task to the current task collection with operator.Options and SSHConnectionProps +func NewSimpleUerSSH(logger *logprinter.Logger, host string, port int, user string, gOpt operator.Options, p *tui.SSHConnectionProps, sshType executor.SSHType) *Builder { + return NewBuilder(logger). + UserSSH( + host, + port, + user, + gOpt.SSHTimeout, + gOpt.OptTimeout, + gOpt.SSHProxyHost, + gOpt.SSHProxyPort, + gOpt.SSHProxyUser, + p.Password, + p.IdentityFile, + p.IdentityFilePassphrase, + gOpt.SSHProxyTimeout, + gOpt.SSHType, + sshType, + ) +} + // UserSSH append a UserSSH task to the current task collection func (b *Builder) UserSSH( host string, port int, deployUser string, sshTimeout, exeTimeout uint64, diff --git a/pkg/cluster/task/tls.go b/pkg/cluster/task/tls.go index 367a7c1636..956148e34c 100644 --- a/pkg/cluster/task/tls.go +++ b/pkg/cluster/task/tls.go @@ -102,21 +102,21 @@ func (c *TLSCert) Execute(ctx context.Context) error { return ErrNoExecutor } if err := e.Transfer(ctx, caFile, - filepath.Join(c.paths.Deploy, "tls", spec.TLSCACert), + filepath.Join(c.paths.Deploy, spec.TLSCertKeyDir, spec.TLSCACert), false, /* download */ 0, /* limit */ false /* compress */); err != nil { return errors.Annotate(err, "failed to transfer CA cert to server") } if err := e.Transfer(ctx, keyFile, - filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.pem", c.role)), + filepath.Join(c.paths.Deploy, spec.TLSCertKeyDir, fmt.Sprintf("%s.pem", c.role)), false, /* download */ 0, /* limit */ false /* compress */); err != nil { return errors.Annotate(err, "failed to transfer TLS private key to server") } if err := e.Transfer(ctx, certFile, - filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.crt", c.role)), + filepath.Join(c.paths.Deploy, spec.TLSCertKeyDir, fmt.Sprintf("%s.crt", c.role)), false, /* download */ 0, /* limit */ false /* compress */); err != nil {