From 6e8d6e25a84a96672779b90ffdf6105e6cc180f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Qi=CE=BC=24hi=D0=AFu=C3=AD?= <39378935+srstack@users.noreply.github.com> Date: Mon, 18 Apr 2022 12:00:03 +0800 Subject: [PATCH 1/7] tiup: supprot record tiup execution history (#1808) --- cmd/history.go | 107 ++++++++++++++++ cmd/root.go | 7 ++ go.mod | 1 + pkg/environment/history.go | 244 +++++++++++++++++++++++++++++++++++++ 4 files changed, 359 insertions(+) create mode 100644 cmd/history.go create mode 100644 pkg/environment/history.go diff --git a/cmd/history.go b/cmd/history.go new file mode 100644 index 0000000000..8adc2b18c9 --- /dev/null +++ b/cmd/history.go @@ -0,0 +1,107 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/environment" + "github.com/pingcap/tiup/pkg/tui" + "github.com/spf13/cobra" +) + +// newHistoryCmd history +func newHistoryCmd() *cobra.Command { + rows := 100 + var displayMode string + var all bool + cmd := &cobra.Command{ + Use: "history ", + Short: "Display the historical execution record of TiUP, displays 100 lines by default", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + r, err := strconv.Atoi(args[0]) + if err == nil { + rows = r + } else { + return fmt.Errorf("%s: numeric argument required", args[0]) + } + } + + env := environment.GlobalEnv() + rows, err := env.GetHistory(rows, all) + if err != nil { + return err + } + + if displayMode == "json" { + for _, r := range rows { + rBytes, err := json.Marshal(r) + if err != nil { + continue + } + fmt.Println(string(rBytes)) + } + return nil + } + var table [][]string + table = append(table, []string{"Date", "Command", "Code"}) + + for _, r := range rows { + table = append(table, []string{ + r.Date.Format("2006-01-02T15:04:05"), + r.Command, + strconv.Itoa(r.Code), + }) + } + tui.PrintTable(table, true) + fmt.Printf("history log save path: %s\n", env.LocalPath(environment.HistoryDir)) + return nil + }, + } + cmd.Flags().StringVar(&displayMode, "format", "default", "The format of output, available values are [default, json]") + cmd.Flags().BoolVar(&all, "all", false, "Display all execution history") + cmd.AddCommand(newHistoryCleanupCmd()) + return cmd +} + +func newHistoryCleanupCmd() *cobra.Command { + var retainDays int + var all bool + var skipConfirm bool + cmd := &cobra.Command{ + Use: "cleanup", + Short: "delete all execution history", + RunE: func(cmd *cobra.Command, args []string) error { + if retainDays < 0 { + return errors.Errorf("retain-days cannot be less than 0") + } + + if all { + retainDays = 0 + } + + env := environment.GlobalEnv() + return env.DeleteHistory(retainDays, skipConfirm) + }, + } + + cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep history for deletion") + cmd.Flags().BoolVar(&all, "all", false, "Delete all history") + cmd.Flags().BoolVarP(&skipConfirm, "yes", "y", false, "Skip all confirmations and assumes 'yes'") + return cmd +} diff --git a/cmd/root.go b/cmd/root.go index c8d3596072..c3afc690a8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -198,6 +198,7 @@ the latest stable version will be downloaded from the repository.`, newMirrorCmd(), newTelemetryCmd(), newEnvCmd(), + newHistoryCmd(), ) originHelpFunc := rootCmd.HelpFunc() @@ -248,6 +249,12 @@ func Execute() { // us a dedicated package for that reportEnabled = false } else { + // record TiUP execution history + err := environment.HistoryRecord(env, os.Args, start, code) + if err != nil { + log.Warnf("Record TiUP execution history log failed: %v", err) + } + teleMeta, _, err := telemetry.GetMeta(env) if err == nil { reportEnabled = teleMeta.Status == telemetry.EnableStatus diff --git a/go.mod b/go.mod index 4a78de0974..3653b1c439 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20220125073028-58f2ac94aa38 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect github.com/pingcap/tidb-insight/collector v0.0.0-20220111101533-227008e9835b + github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.32.1 github.com/prometheus/prom2json v1.3.0 diff --git a/pkg/environment/history.go b/pkg/environment/history.go new file mode 100644 index 0000000000..a71e303fb7 --- /dev/null +++ b/pkg/environment/history.go @@ -0,0 +1,244 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + "github.com/pingcap/tiup/pkg/tui" + "github.com/pingcap/tiup/pkg/utils" + "github.com/pkg/errors" +) + +const ( + // HistoryDir history save path + HistoryDir = "history" + historyPrefix = "tiup-history-" + historySize int64 = 1024 * 64 // history file default size is 64k +) + +// commandRow type of command history row +type historyRow struct { + Date time.Time `json:"time"` + Command string `json:"command"` + Code int `json:"exit_code"` +} + +// historyItem record history row file item +type historyItem struct { + path string + info fs.FileInfo + index int +} + +// HistoryRecord record tiup exec cmd +func HistoryRecord(env *Environment, command []string, date time.Time, code int) error { + if env == nil { + return nil + } + + historyPath := env.LocalPath(HistoryDir) + if utils.IsNotExist(historyPath) { + err := os.MkdirAll(historyPath, 0755) + if err != nil { + return err + } + } + + h := &historyRow{ + Command: strings.Join(command, " "), + Date: date, + Code: code, + } + + return h.save(historyPath) +} + +// save save commandRow to file +func (r *historyRow) save(dir string) error { + rBytes, err := json.Marshal(r) + if err != nil { + return err + } + + historyFile := getLatestHistoryFile(dir) + + f, err := os.OpenFile(historyFile.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(append(rBytes, []byte("\n")...)) + return err +} + +// GetHistory get tiup history +func (env *Environment) GetHistory(count int, all bool) ([]*historyRow, error) { + fList, err := getHistoryFileList(env.LocalPath(HistoryDir)) + if err != nil { + return nil, err + } + rows := []*historyRow{} + for _, f := range fList { + rs, err := f.getHistory() + if err != nil { + return rows, err + } + if (len(rows)+len(rs)) > count && !all { + i := len(rows) + len(rs) - count + rows = append(rs[i:], rows...) + break + } + + rows = append(rs, rows...) + } + return rows, nil +} + +// DeleteHistory delete history file +func (env *Environment) DeleteHistory(retainDays int, skipConfirm bool) error { + if retainDays < 0 { + return errors.Errorf("retainDays cannot be less than 0") + } + + // history file before `DelBeforeTime` will be deleted + oneDayDuration, _ := time.ParseDuration("-24h") + delBeforeTime := time.Now().Add(oneDayDuration * time.Duration(retainDays)) + + if !skipConfirm { + fmt.Printf("History logs before %s will be %s!\n", + color.HiYellowString(delBeforeTime.Format("2006-01-02T15:04:05")), + color.HiYellowString("deleted"), + ) + if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { + return err + } + } + + fList, err := getHistoryFileList(env.LocalPath(HistoryDir)) + if err != nil { + return err + } + + if len(fList) == 0 { + return nil + } + + for _, f := range fList { + if f.info.ModTime().Before(delBeforeTime) { + err := os.Remove(f.path) + if err != nil { + return err + } + continue + } + } + return nil +} + +// getHistory get tiup history execution row +func (i *historyItem) getHistory() ([]*historyRow, error) { + rows := []*historyRow{} + + fi, err := os.Open(i.path) + if err != nil { + return rows, err + } + defer fi.Close() + + br := bufio.NewReader(fi) + for { + a, _, c := br.ReadLine() + if c == io.EOF { + break + } + r := &historyRow{} + // ignore + err := json.Unmarshal(a, r) + if err != nil { + continue + } + rows = append(rows, r) + } + + return rows, nil +} + +// getHistoryFileList get the history file list +func getHistoryFileList(dir string) ([]historyItem, error) { + fileInfos, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + hfileList := []historyItem{} + for _, fi := range fileInfos { + if fi.IsDir() { + continue + } + + // another suffix + // ex: tiup-history-0.bak + i, err := strconv.Atoi((strings.TrimPrefix(fi.Name(), historyPrefix))) + if err != nil { + continue + } + + fInfo, _ := fi.Info() + hfileList = append(hfileList, historyItem{ + path: filepath.Join(dir, fi.Name()), + index: i, + info: fInfo, + }) + } + + sort.Slice(hfileList, func(i, j int) bool { + return hfileList[i].index > hfileList[j].index + }) + + return hfileList, nil +} + +// getLatestHistoryFile get the latest history file, use index 0 if it doesn't exist +func getLatestHistoryFile(dir string) (item historyItem) { + fileList, err := getHistoryFileList(dir) + // start from 0 + if len(fileList) == 0 || err != nil { + item.index = 0 + item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) + return + } + + latestItem := fileList[0] + + if latestItem.info.Size() >= historySize { + item.index = latestItem.index + 1 + item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) + } else { + item = latestItem + } + + return +} From e4757455f6f6583e4916182dce7757a0a3438f24 Mon Sep 17 00:00:00 2001 From: nexustar Date: Mon, 18 Apr 2022 12:36:03 +0800 Subject: [PATCH 2/7] tiup: refactor help (#1831) --- cmd/help.go | 148 ----------------------------------- cmd/root.go | 123 +++++++++++++++-------------- pkg/repository/repository.go | 1 - 3 files changed, 61 insertions(+), 211 deletions(-) delete mode 100644 cmd/help.go diff --git a/cmd/help.go b/cmd/help.go deleted file mode 100644 index da6bf175d7..0000000000 --- a/cmd/help.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pingcap/tiup/pkg/environment" - "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/utils" - "github.com/spf13/cobra" -) - -func newHelpCmd() *cobra.Command { - return &cobra.Command{ - Use: "help [command]", - Short: "Help about any command or component", - Long: `Help provides help for any command or component in the application. -Simply type tiup help | for full details.`, - Run: func(cmd *cobra.Command, args []string) { - env := environment.GlobalEnv() - cmd, n, e := cmd.Root().Find(args) - if (cmd == rootCmd || e != nil) && len(n) > 0 { - externalHelp(env, n[0], n[1:]...) - } else { - cmd.InitDefaultHelpFlag() // make possible 'help' flag to be shown - cmd.HelpFunc()(cmd, args) - } - }, - } -} - -func externalHelp(env *environment.Environment, spec string, args ...string) { - profile := env.Profile() - component, version := environment.ParseCompVersion(spec) - selectVer, err := env.SelectInstalledVersion(component, version) - if err != nil { - fmt.Println(err) - return - } - binaryPath, err := env.BinaryPath(component, selectVer) - if err != nil { - fmt.Println(err) - return - } - - installPath, err := profile.ComponentInstalledPath(component, selectVer) - if err != nil { - fmt.Println(err) - return - } - - sd := profile.Path(filepath.Join(localdata.StorageParentDir, strings.Split(spec, ":")[0])) - envs := []string{ - fmt.Sprintf("%s=%s", localdata.EnvNameHome, profile.Root()), - fmt.Sprintf("%s=%s", localdata.EnvNameComponentInstallDir, installPath), - fmt.Sprintf("%s=%s", localdata.EnvNameComponentDataDir, sd), - } - envs = append(envs, os.Environ()...) - - comp := exec.Command(binaryPath, utils.RebuildArgs(args)...) - comp.Env = envs - comp.Stdout = os.Stdout - comp.Stderr = os.Stderr - if err := comp.Start(); err != nil { - fmt.Printf("Cannot fetch help message from %s failed: %v\n", binaryPath, err) - return - } - if err := comp.Wait(); err != nil { - fmt.Printf("Cannot fetch help message from %s failed: %v\n", binaryPath, err) - } -} - -// nolint unused -func rebuildArgs(args []string) []string { - helpFlag := "--help" - argList := []string{} - for _, arg := range args { - if arg == "-h" || arg == "--help" { - helpFlag = arg - } else { - argList = append(argList, arg) - } - } - argList = append(argList, helpFlag) - return argList -} - -func usageTemplate(profile *localdata.Profile) string { - installComps := ` -Components Manifest: - use "tiup list" to fetch the latest components manifest -` - - return `Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} -{{if not .HasParent}}` + installComps + `{{end}} -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Global Flags: -{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} - -Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} - {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if not .HasParent}} - -Component instances with the same "tag" will share a data directory ($TIUP_HOME/data/$tag): - $ tiup --tag mycluster playground - -Examples: - $ tiup playground # Quick start - $ tiup playground nightly # Start a playground with the latest nightly version - $ tiup install [:version] # Install a component of specific version - $ tiup update --all # Update all installed components to the latest version - $ tiup update --nightly # Update all installed components to the nightly version - $ tiup update --self # Update the "tiup" to the latest version - $ tiup list # Fetch the latest supported components list - $ tiup status # Display all running/terminated instances - $ tiup clean # Clean the data of running/terminated instance (Kill process if it's running) - $ tiup clean --all # Clean the data of all running/terminated instances{{end}}{{if .HasAvailableSubCommands}} - -Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} -` -} diff --git a/cmd/root.go b/cmd/root.go index c3afc690a8..d1e4410317 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -49,10 +49,8 @@ var ( // arguments var ( - binary string - binPath string - tag string - printVersion bool // not using cobra.Command.Version to make it possible to show component versions + binPath string + tag string ) func init() { @@ -67,18 +65,25 @@ TiDB platform components to the local system. You can run a specific version of "tiup [:version]". If no version number is specified, the latest version installed locally will be used. If the specified component does not have any version installed locally, the latest stable version will be downloaded from the repository.`, + Example: ` $ tiup playground # Quick start + $ tiup playground nightly # Start a playground with the latest nightly version + $ tiup install [:version] # Install a component of specific version + $ tiup update --all # Update all installed components to the latest version + $ tiup update --nightly # Update all installed components to the nightly version + $ tiup update --self # Update the "tiup" to the latest version + $ tiup list # Fetch the latest supported components list + $ tiup status # Display all running/terminated instances + $ tiup clean # Clean the data of running/terminated instance (Kill process if it's running) + $ tiup clean --all # Clean the data of all running/terminated instances`, SilenceErrors: true, - FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true}, + DisableFlagParsing: true, Args: func(cmd *cobra.Command, args []string) error { // Support `tiup ` return nil }, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { teleCommand = cmd.CommandPath() - if printVersion && len(args) == 0 { - return nil - } switch cmd.Name() { case "init", "set": @@ -100,13 +105,30 @@ the latest stable version will be downloaded from the repository.`, return nil }, RunE: func(cmd *cobra.Command, args []string) error { - if printVersion && len(args) == 0 { - fmt.Println(version.NewTiUPVersion().String()) - return nil + if len(args) == 0 { + return cmd.Help() } env := environment.GlobalEnv() - if binary != "" { - component, ver := environment.ParseCompVersion(binary) + + // TBD: change this flag to subcommand + + // We assume the first unknown parameter is the component name and following + // parameters will be transparent passed because registered flags and subcommands + // will be parsed correctly. + // e.g: tiup --tag mytag --rm playground --db 3 --pd 3 --kv 4 + // => run "playground" with parameters "--db 3 --pd 3 --kv 4" + // tiup --tag mytag --binpath /xxx/tikv-server tikv + switch args[0] { + case "--help", "-h": + return cmd.Help() + case "--version", "-v": + fmt.Println(version.NewTiUPVersion().String()) + return nil + case "--binary": + if len(args) < 2 { + return fmt.Errorf("flag needs an argument: %s", args[0]) + } + component, ver := environment.ParseCompVersion(args[1]) selectedVer, err := env.SelectInstalledVersion(component, ver) if err != nil { return err @@ -117,36 +139,31 @@ the latest stable version will be downloaded from the repository.`, } fmt.Println(binaryPath) return nil - } - if len(args) > 0 { - // We assume the first unknown parameter is the component name and following - // parameters will be transparent passed because registered flags and subcommands - // will be parsed correctly. - // e.g: tiup --tag mytag --rm playground --db 3 --pd 3 --kv 4 - // => run "playground" with parameters "--db 3 --pd 3 --kv 4" - // tiup --tag mytag --binpath /xxx/tikv-server tikv - var transparentParams []string - componentSpec := args[0] - for i, arg := range os.Args { - if arg == componentSpec { - transparentParams = os.Args[i+1:] - break - } + case "--binpath": + if len(args) < 2 { + return fmt.Errorf("flag needs an argument: %s", args[0]) } - if len(transparentParams) > 0 && transparentParams[0] == "--" { - transparentParams = transparentParams[1:] + binPath = args[1] + args = args[2:] + case "--tag", "-T": + if len(args) < 2 { + return fmt.Errorf("flag needs an argument: %s", args[0]) } - - teleCommand = fmt.Sprintf("%s %s", cmd.CommandPath(), componentSpec) - return tiupexec.RunComponent(env, tag, componentSpec, binPath, transparentParams) + tag = args[1] + args = args[2:] } - return cmd.Help() - }, - PersistentPostRunE: func(cmd *cobra.Command, args []string) error { - if env := environment.GlobalEnv(); env != nil { - return env.Close() + if len(args) < 1 { + return cmd.Help() } - return nil + + componentSpec := args[0] + args = args[1:] + if len(args) > 0 && args[0] == "--" { + args = args[1:] + } + + teleCommand = fmt.Sprintf("%s %s", cmd.CommandPath(), componentSpec) + return tiupexec.RunComponent(env, tag, componentSpec, binPath, args) }, SilenceUsage: true, // implement auto completion for tiup components @@ -181,12 +198,12 @@ the latest stable version will be downloaded from the repository.`, }, } - rootCmd.PersistentFlags().BoolVarP(&repoOpts.SkipVersionCheck, "skip-version-check", "", false, "Skip the strict version check, by default a version must be a valid SemVer string") - rootCmd.Flags().StringVar(&binary, "binary", "", "Print binary path of a specific version of a component `[:version]`\n"+ + // useless, exist to generate help infomation + rootCmd.Flags().String("binary", "", "Print binary path of a specific version of a component `[:version]`\n"+ "and the latest version installed will be selected if no version specified") - rootCmd.Flags().StringVarP(&tag, "tag", "T", "", "[Deprecated] Specify a tag for component instance") - rootCmd.Flags().StringVar(&binPath, "binpath", "", "Specify the binary path of component instance") - rootCmd.Flags().BoolVarP(&printVersion, "version", "v", false, "Print the version of tiup") + rootCmd.Flags().StringP("tag", "T", "", "[Deprecated] Specify a tag for component instance") + rootCmd.Flags().String("binpath", "", "Specify the binary path of component instance") + rootCmd.Flags().BoolP("version", "v", false, "Print the version of tiup") rootCmd.AddCommand( newInstallCmd(), @@ -200,24 +217,6 @@ the latest stable version will be downloaded from the repository.`, newEnvCmd(), newHistoryCmd(), ) - - originHelpFunc := rootCmd.HelpFunc() - rootCmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { - cmd, _, _ = cmd.Root().Find(args) - if len(args) < 2 || cmd != rootCmd { - originHelpFunc(cmd, args) - return - } - - env, _ := environment.InitEnv(repoOpts) - environment.SetGlobalEnv(env) - _ = cmd.RunE(cmd, args) - }) - - rootCmd.SetHelpCommand(newHelpCmd()) - // If env is inited before, localdata.InitProfile() will return a valid profile - // or it will return an invalid one but still print usage - rootCmd.SetUsageTemplate(usageTemplate(localdata.InitProfile())) } // Execute parses the command line arguments and calls proper functions diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index 8924ace8ef..0dd38fb3e1 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -22,7 +22,6 @@ type Repository struct { // Options represents options for a repository type Options struct { - SkipVersionCheck bool GOOS string GOARCH string DisableDecompress bool From a1f44d72c6d3e45af2dcc40d9557f849f66683bd Mon Sep 17 00:00:00 2001 From: glkappe Date: Mon, 18 Apr 2022 17:02:03 +0800 Subject: [PATCH 3/7] add altermanager config (#1818) --- embed/examples/cluster/topology.example.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/embed/examples/cluster/topology.example.yaml b/embed/examples/cluster/topology.example.yaml index 0edac37932..c81f4824fe 100644 --- a/embed/examples/cluster/topology.example.yaml +++ b/embed/examples/cluster/topology.example.yaml @@ -327,3 +327,5 @@ alertmanager_servers: # data_dir: "/tidb-data/alertmanager-9093" # # Alertmanager log file storage directory. # log_dir: "/tidb-deploy/alertmanager-9093/log" + # # Alertmanager config file storage directory. + # config_file: "/tidb-deploy/alertmanager-9093/bin/alertmanager/alertmanager.yml" From 0567f77835c8ea0d975a8fbb52df68773961ae80 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Mon, 18 Apr 2022 17:26:03 +0800 Subject: [PATCH 4/7] mirror: support custom key-dir when rotate root.json (#1848) --- cmd/mirror.go | 12 ++++++++++++ cmd/root.go | 5 +++-- components/cluster/command/root.go | 2 +- components/dm/command/root.go | 2 +- components/playground/main.go | 2 +- pkg/environment/env.go | 4 ++-- pkg/repository/model/model.go | 12 ++++++++++++ 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/cmd/mirror.go b/cmd/mirror.go index 171ca15295..4c9ffe17aa 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -444,6 +444,7 @@ func newTransferOwnerCmd() *cobra.Command { // the `mirror rotate` sub command func newMirrorRotateCmd() *cobra.Command { addr := "0.0.0.0:8080" + keyDir := "" cmd := &cobra.Command{ Use: "rotate", @@ -451,6 +452,16 @@ func newMirrorRotateCmd() *cobra.Command { Long: "Rotate root.json make it possible to modify root.json", RunE: func(cmd *cobra.Command, args []string) error { teleCommand = cmd.CommandPath() + + e, err := environment.InitEnv(repoOpts, repository.MirrorOptions{KeyDir: keyDir}) + if err != nil { + if errors.Is(perrs.Cause(err), v1manifest.ErrLoadManifest) { + log.Warnf("Please check for root manifest file, you may download one from the repository mirror, or try `tiup mirror set` to force reset it.") + } + return err + } + environment.SetGlobalEnv(e) + root, err := editLatestRootManifest() if err != nil { return err @@ -465,6 +476,7 @@ func newMirrorRotateCmd() *cobra.Command { }, } cmd.Flags().StringVarP(&addr, "addr", "", addr, "listen address:port when starting the temp server for rotating") + cmd.Flags().StringVarP(&keyDir, "key-dir", "", keyDir, "specify the directory where stores the private keys") return cmd } diff --git a/cmd/root.go b/cmd/root.go index d1e4410317..e9422ced8a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -86,6 +86,7 @@ the latest stable version will be downloaded from the repository.`, teleCommand = cmd.CommandPath() switch cmd.Name() { case "init", + "rotate", "set": if cmd.HasParent() && cmd.Parent().Name() == "mirror" { // skip environment init @@ -93,7 +94,7 @@ the latest stable version will be downloaded from the repository.`, } fallthrough default: - e, err := environment.InitEnv(repoOpts) + e, err := environment.InitEnv(repoOpts, repository.MirrorOptions{}) if err != nil { if errors.Is(perrs.Cause(err), v1manifest.ErrLoadManifest) { log.Warnf("Please check for root manifest file, you may download one from the repository mirror, or try `tiup mirror set` to force reset it.") @@ -198,7 +199,7 @@ the latest stable version will be downloaded from the repository.`, }, } - // useless, exist to generate help infomation + // useless, exist to generate help information rootCmd.Flags().String("binary", "", "Print binary path of a specific version of a component `[:version]`\n"+ "and the latest version installed will be selected if no version specified") rootCmd.Flags().StringP("tag", "T", "", "[Deprecated] Specify a tag for component instance") diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go index ca976c7e92..6bad82f4fc 100644 --- a/components/cluster/command/root.go +++ b/components/cluster/command/root.go @@ -120,7 +120,7 @@ func init() { env, err = tiupmeta.InitEnv(repository.Options{ GOOS: "linux", GOARCH: "amd64", - }) + }, repository.MirrorOptions{}) if err != nil { return err } diff --git a/components/dm/command/root.go b/components/dm/command/root.go index 4246c5088f..b6bf79ac9c 100644 --- a/components/dm/command/root.go +++ b/components/dm/command/root.go @@ -91,7 +91,7 @@ please backup your data before process.`, env, err = tiupmeta.InitEnv(repository.Options{ GOOS: "linux", GOARCH: "amd64", - }) + }, repository.MirrorOptions{}) if err != nil { return err } diff --git a/components/playground/main.go b/components/playground/main.go index a2c159733b..bdade14803 100644 --- a/components/playground/main.go +++ b/components/playground/main.go @@ -218,7 +218,7 @@ Examples: return err } - env, err := environment.InitEnv(repository.Options{}) + env, err := environment.InitEnv(repository.Options{}, repository.MirrorOptions{}) if err != nil { return err } diff --git a/pkg/environment/env.go b/pkg/environment/env.go index a2d35f69bf..4bb55dd368 100644 --- a/pkg/environment/env.go +++ b/pkg/environment/env.go @@ -81,7 +81,7 @@ type Environment struct { } // InitEnv creates a new Environment object configured using env vars and defaults. -func InitEnv(options repository.Options) (*Environment, error) { +func InitEnv(options repository.Options, mOpt repository.MirrorOptions) (*Environment, error) { if env := GlobalEnv(); env != nil { return env, nil } @@ -92,7 +92,7 @@ func InitEnv(options repository.Options) (*Environment, error) { // Initialize the repository // Replace the mirror if some sub-commands use different mirror address mirrorAddr := Mirror() - mirror := repository.NewMirror(mirrorAddr, repository.MirrorOptions{}) + mirror := repository.NewMirror(mirrorAddr, mOpt) if err := mirror.Open(); err != nil { return nil, err } diff --git a/pkg/repository/model/model.go b/pkg/repository/model/model.go index 5024cdc74c..6352adc302 100644 --- a/pkg/repository/model/model.go +++ b/pkg/repository/model/model.go @@ -134,10 +134,22 @@ func (m *model) Rotate(manifest *v1manifest.Manifest) error { return err } + // write new 'root.json' file with verion prefix manifestFilename := fmt.Sprintf("%d.root.json", root.Version) if err := m.txn.WriteManifest(manifestFilename, manifest); err != nil { return err } + /* not yet update the 'root.json' without version prefix, as we don't + * have a '1.root.json', so the 'root.json' is playing the role of initial + * '1.root.json', clients are updating to the latest 'n.root.json' no + * matter older ones are expired or not + * maybe we could update the 'root.json' some day when we have many many + * versions of root.json available and the updating process from old clients + * are causing performance issues + */ + // if err := m.txn.WriteManifest("root.json", manifest); err != nil { + // return err + // } fi, err := m.txn.Stat(manifestFilename) if err != nil { From d67729a032a9205e160a89c2a91e0ecb78e27adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Qi=CE=BC=24hi=D0=AFu=C3=AD?= <39378935+srstack@users.noreply.github.com> Date: Tue, 19 Apr 2022 11:50:04 +0800 Subject: [PATCH 5/7] ci: support-auto-update-homebrew (#1844) --- .github/workflows/release-tiup.yaml | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/release-tiup.yaml b/.github/workflows/release-tiup.yaml index a0821e1431..047cca3056 100644 --- a/.github/workflows/release-tiup.yaml +++ b/.github/workflows/release-tiup.yaml @@ -26,6 +26,8 @@ jobs: release: runs-on: ubuntu-latest timeout-minutes: 30 + outputs: + REL_VER: ${{ steps.build_tiup.outputs.REL_VER }} strategy: fail-fast: true matrix: @@ -195,3 +197,40 @@ jobs: omitNameDuringUpdate: true prerelease: ${{ github.event.release.prerelease }} token: ${{ secrets.GITHUB_TOKEN }} + + brew-upgrade: + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: release + steps: + - name: Check out brew code + uses: actions/checkout@v3 + continue-on-error: true + if: github.event_name == 'release' + with: + repository: pingcap/homebrew-brew + persist-credentials: false + ref: master + path: ${{ github.workspace }}/homebrew-brew + fetch-depth: 0 + + - name: Update and Check tiup version + id: update_version + working-directory: ${{ github.workspace }}/homebrew-brew + continue-on-error: true + if: github.event_name == 'release' + run: | + sed -i 's/version.*/version "${{ needs.release.outputs.REL_VER }}"/g' Formula/tiup.rb + sed -i 's/tag:.*/tag: "${{ needs.release.outputs.REL_VER }}"/g' Formula/tiup.rb + cat Formula/tiup.rb + + - name: Push new homebrew + uses: actions-js/push@master + continue-on-error: true + if: github.event_name == 'release' + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + directory: ${{ github.workspace }}/homebrew-brew + message: "tiup: ${{ needs.release.outputs.REL_VER }}" + branch: master + repository: pingcap/homebrew-brew \ No newline at end of file From e5b8861326e6e30181aaff40f289158908a86045 Mon Sep 17 00:00:00 2001 From: nexustar Date: Tue, 19 Apr 2022 14:20:04 +0800 Subject: [PATCH 6/7] dm: bind to 0.0.0.0 (#1845) --- embed/templates/scripts/run_dm-master.sh.tpl | 2 +- embed/templates/scripts/run_dm-master_scale.sh.tpl | 2 +- embed/templates/scripts/run_dm-worker.sh.tpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/embed/templates/scripts/run_dm-master.sh.tpl b/embed/templates/scripts/run_dm-master.sh.tpl index 246e2097ad..39fde5e418 100644 --- a/embed/templates/scripts/run_dm-master.sh.tpl +++ b/embed/templates/scripts/run_dm-master.sh.tpl @@ -25,7 +25,7 @@ exec bin/dm-master/dm-master \ --v1-sources-path="{{.V1SourcePath}}" \ {{- end}} --name="{{.Name}}" \ - --master-addr="{{.IP}}:{{.Port}}" \ + --master-addr="0.0.0.0:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ diff --git a/embed/templates/scripts/run_dm-master_scale.sh.tpl b/embed/templates/scripts/run_dm-master_scale.sh.tpl index 90572b0241..13a31977aa 100644 --- a/embed/templates/scripts/run_dm-master_scale.sh.tpl +++ b/embed/templates/scripts/run_dm-master_scale.sh.tpl @@ -22,7 +22,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-master/d exec bin/dm-master/dm-master \ {{- end}} --name="{{.Name}}" \ - --master-addr="{{.IP}}:{{.Port}}" \ + --master-addr="0.0.0.0:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ diff --git a/embed/templates/scripts/run_dm-worker.sh.tpl b/embed/templates/scripts/run_dm-worker.sh.tpl index df11e96e20..f1074b4967 100644 --- a/embed/templates/scripts/run_dm-worker.sh.tpl +++ b/embed/templates/scripts/run_dm-worker.sh.tpl @@ -23,7 +23,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-worker/d exec bin/dm-worker/dm-worker \ {{- end}} --name="{{.Name}}" \ - --worker-addr="{{.IP}}:{{.Port}}" \ + --worker-addr="0.0.0.0:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --log-file="{{.LogDir}}/dm-worker.log" \ --join="{{template "MasterList" .Endpoints}}" \ From 762dd9486218e3d1248a1f93a02bfec41d712f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Qi=CE=BC=24hi=D0=AFu=C3=AD?= <39378935+srstack@users.noreply.github.com> Date: Wed, 20 Apr 2022 15:58:03 +0800 Subject: [PATCH 7/7] fix: cluster prune will destroy pump/drainer directly after scale-in (#1851) --- pkg/cluster/operation/destroy.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index c07c3a3b1e..b92a62200c 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -600,6 +600,7 @@ func DestroyClusterTombstone( if !tombstone { pumpServers = append(pumpServers, s) + continue } nodes = append(nodes, id) @@ -629,6 +630,7 @@ func DestroyClusterTombstone( if !tombstone { drainerServers = append(drainerServers, s) + continue } nodes = append(nodes, id)