diff --git a/.github/workflows/release-tiup.yaml b/.github/workflows/release-tiup.yaml index a0821e1431..047cca3056 100644 --- a/.github/workflows/release-tiup.yaml +++ b/.github/workflows/release-tiup.yaml @@ -26,6 +26,8 @@ jobs: release: runs-on: ubuntu-latest timeout-minutes: 30 + outputs: + REL_VER: ${{ steps.build_tiup.outputs.REL_VER }} strategy: fail-fast: true matrix: @@ -195,3 +197,40 @@ jobs: omitNameDuringUpdate: true prerelease: ${{ github.event.release.prerelease }} token: ${{ secrets.GITHUB_TOKEN }} + + brew-upgrade: + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: release + steps: + - name: Check out brew code + uses: actions/checkout@v3 + continue-on-error: true + if: github.event_name == 'release' + with: + repository: pingcap/homebrew-brew + persist-credentials: false + ref: master + path: ${{ github.workspace }}/homebrew-brew + fetch-depth: 0 + + - name: Update and Check tiup version + id: update_version + working-directory: ${{ github.workspace }}/homebrew-brew + continue-on-error: true + if: github.event_name == 'release' + run: | + sed -i 's/version.*/version "${{ needs.release.outputs.REL_VER }}"/g' Formula/tiup.rb + sed -i 's/tag:.*/tag: "${{ needs.release.outputs.REL_VER }}"/g' Formula/tiup.rb + cat Formula/tiup.rb + + - name: Push new homebrew + uses: actions-js/push@master + continue-on-error: true + if: github.event_name == 'release' + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + directory: ${{ github.workspace }}/homebrew-brew + message: "tiup: ${{ needs.release.outputs.REL_VER }}" + branch: master + repository: pingcap/homebrew-brew \ No newline at end of file diff --git a/cmd/help.go b/cmd/help.go deleted file mode 100644 index da6bf175d7..0000000000 --- a/cmd/help.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pingcap/tiup/pkg/environment" - "github.com/pingcap/tiup/pkg/localdata" - "github.com/pingcap/tiup/pkg/utils" - "github.com/spf13/cobra" -) - -func newHelpCmd() *cobra.Command { - return &cobra.Command{ - Use: "help [command]", - Short: "Help about any command or component", - Long: `Help provides help for any command or component in the application. -Simply type tiup help | for full details.`, - Run: func(cmd *cobra.Command, args []string) { - env := environment.GlobalEnv() - cmd, n, e := cmd.Root().Find(args) - if (cmd == rootCmd || e != nil) && len(n) > 0 { - externalHelp(env, n[0], n[1:]...) - } else { - cmd.InitDefaultHelpFlag() // make possible 'help' flag to be shown - cmd.HelpFunc()(cmd, args) - } - }, - } -} - -func externalHelp(env *environment.Environment, spec string, args ...string) { - profile := env.Profile() - component, version := environment.ParseCompVersion(spec) - selectVer, err := env.SelectInstalledVersion(component, version) - if err != nil { - fmt.Println(err) - return - } - binaryPath, err := env.BinaryPath(component, selectVer) - if err != nil { - fmt.Println(err) - return - } - - installPath, err := profile.ComponentInstalledPath(component, selectVer) - if err != nil { - fmt.Println(err) - return - } - - sd := profile.Path(filepath.Join(localdata.StorageParentDir, strings.Split(spec, ":")[0])) - envs := []string{ - fmt.Sprintf("%s=%s", localdata.EnvNameHome, profile.Root()), - fmt.Sprintf("%s=%s", localdata.EnvNameComponentInstallDir, installPath), - fmt.Sprintf("%s=%s", localdata.EnvNameComponentDataDir, sd), - } - envs = append(envs, os.Environ()...) - - comp := exec.Command(binaryPath, utils.RebuildArgs(args)...) - comp.Env = envs - comp.Stdout = os.Stdout - comp.Stderr = os.Stderr - if err := comp.Start(); err != nil { - fmt.Printf("Cannot fetch help message from %s failed: %v\n", binaryPath, err) - return - } - if err := comp.Wait(); err != nil { - fmt.Printf("Cannot fetch help message from %s failed: %v\n", binaryPath, err) - } -} - -// nolint unused -func rebuildArgs(args []string) []string { - helpFlag := "--help" - argList := []string{} - for _, arg := range args { - if arg == "-h" || arg == "--help" { - helpFlag = arg - } else { - argList = append(argList, arg) - } - } - argList = append(argList, helpFlag) - return argList -} - -func usageTemplate(profile *localdata.Profile) string { - installComps := ` -Components Manifest: - use "tiup list" to fetch the latest components manifest -` - - return `Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} -{{if not .HasParent}}` + installComps + `{{end}} -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Global Flags: -{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} - -Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} - {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if not .HasParent}} - -Component instances with the same "tag" will share a data directory ($TIUP_HOME/data/$tag): - $ tiup --tag mycluster playground - -Examples: - $ tiup playground # Quick start - $ tiup playground nightly # Start a playground with the latest nightly version - $ tiup install [:version] # Install a component of specific version - $ tiup update --all # Update all installed components to the latest version - $ tiup update --nightly # Update all installed components to the nightly version - $ tiup update --self # Update the "tiup" to the latest version - $ tiup list # Fetch the latest supported components list - $ tiup status # Display all running/terminated instances - $ tiup clean # Clean the data of running/terminated instance (Kill process if it's running) - $ tiup clean --all # Clean the data of all running/terminated instances{{end}}{{if .HasAvailableSubCommands}} - -Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} -` -} diff --git a/cmd/history.go b/cmd/history.go new file mode 100644 index 0000000000..8adc2b18c9 --- /dev/null +++ b/cmd/history.go @@ -0,0 +1,107 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/environment" + "github.com/pingcap/tiup/pkg/tui" + "github.com/spf13/cobra" +) + +// newHistoryCmd history +func newHistoryCmd() *cobra.Command { + rows := 100 + var displayMode string + var all bool + cmd := &cobra.Command{ + Use: "history ", + Short: "Display the historical execution record of TiUP, displays 100 lines by default", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + r, err := strconv.Atoi(args[0]) + if err == nil { + rows = r + } else { + return fmt.Errorf("%s: numeric argument required", args[0]) + } + } + + env := environment.GlobalEnv() + rows, err := env.GetHistory(rows, all) + if err != nil { + return err + } + + if displayMode == "json" { + for _, r := range rows { + rBytes, err := json.Marshal(r) + if err != nil { + continue + } + fmt.Println(string(rBytes)) + } + return nil + } + var table [][]string + table = append(table, []string{"Date", "Command", "Code"}) + + for _, r := range rows { + table = append(table, []string{ + r.Date.Format("2006-01-02T15:04:05"), + r.Command, + strconv.Itoa(r.Code), + }) + } + tui.PrintTable(table, true) + fmt.Printf("history log save path: %s\n", env.LocalPath(environment.HistoryDir)) + return nil + }, + } + cmd.Flags().StringVar(&displayMode, "format", "default", "The format of output, available values are [default, json]") + cmd.Flags().BoolVar(&all, "all", false, "Display all execution history") + cmd.AddCommand(newHistoryCleanupCmd()) + return cmd +} + +func newHistoryCleanupCmd() *cobra.Command { + var retainDays int + var all bool + var skipConfirm bool + cmd := &cobra.Command{ + Use: "cleanup", + Short: "delete all execution history", + RunE: func(cmd *cobra.Command, args []string) error { + if retainDays < 0 { + return errors.Errorf("retain-days cannot be less than 0") + } + + if all { + retainDays = 0 + } + + env := environment.GlobalEnv() + return env.DeleteHistory(retainDays, skipConfirm) + }, + } + + cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep history for deletion") + cmd.Flags().BoolVar(&all, "all", false, "Delete all history") + cmd.Flags().BoolVarP(&skipConfirm, "yes", "y", false, "Skip all confirmations and assumes 'yes'") + return cmd +} diff --git a/cmd/mirror.go b/cmd/mirror.go index 171ca15295..4c9ffe17aa 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -444,6 +444,7 @@ func newTransferOwnerCmd() *cobra.Command { // the `mirror rotate` sub command func newMirrorRotateCmd() *cobra.Command { addr := "0.0.0.0:8080" + keyDir := "" cmd := &cobra.Command{ Use: "rotate", @@ -451,6 +452,16 @@ func newMirrorRotateCmd() *cobra.Command { Long: "Rotate root.json make it possible to modify root.json", RunE: func(cmd *cobra.Command, args []string) error { teleCommand = cmd.CommandPath() + + e, err := environment.InitEnv(repoOpts, repository.MirrorOptions{KeyDir: keyDir}) + if err != nil { + if errors.Is(perrs.Cause(err), v1manifest.ErrLoadManifest) { + log.Warnf("Please check for root manifest file, you may download one from the repository mirror, or try `tiup mirror set` to force reset it.") + } + return err + } + environment.SetGlobalEnv(e) + root, err := editLatestRootManifest() if err != nil { return err @@ -465,6 +476,7 @@ func newMirrorRotateCmd() *cobra.Command { }, } cmd.Flags().StringVarP(&addr, "addr", "", addr, "listen address:port when starting the temp server for rotating") + cmd.Flags().StringVarP(&keyDir, "key-dir", "", keyDir, "specify the directory where stores the private keys") return cmd } diff --git a/cmd/root.go b/cmd/root.go index c8d3596072..e9422ced8a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -49,10 +49,8 @@ var ( // arguments var ( - binary string - binPath string - tag string - printVersion bool // not using cobra.Command.Version to make it possible to show component versions + binPath string + tag string ) func init() { @@ -67,20 +65,28 @@ TiDB platform components to the local system. You can run a specific version of "tiup [:version]". If no version number is specified, the latest version installed locally will be used. If the specified component does not have any version installed locally, the latest stable version will be downloaded from the repository.`, + Example: ` $ tiup playground # Quick start + $ tiup playground nightly # Start a playground with the latest nightly version + $ tiup install [:version] # Install a component of specific version + $ tiup update --all # Update all installed components to the latest version + $ tiup update --nightly # Update all installed components to the nightly version + $ tiup update --self # Update the "tiup" to the latest version + $ tiup list # Fetch the latest supported components list + $ tiup status # Display all running/terminated instances + $ tiup clean # Clean the data of running/terminated instance (Kill process if it's running) + $ tiup clean --all # Clean the data of all running/terminated instances`, SilenceErrors: true, - FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true}, + DisableFlagParsing: true, Args: func(cmd *cobra.Command, args []string) error { // Support `tiup ` return nil }, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { teleCommand = cmd.CommandPath() - if printVersion && len(args) == 0 { - return nil - } switch cmd.Name() { case "init", + "rotate", "set": if cmd.HasParent() && cmd.Parent().Name() == "mirror" { // skip environment init @@ -88,7 +94,7 @@ the latest stable version will be downloaded from the repository.`, } fallthrough default: - e, err := environment.InitEnv(repoOpts) + e, err := environment.InitEnv(repoOpts, repository.MirrorOptions{}) if err != nil { if errors.Is(perrs.Cause(err), v1manifest.ErrLoadManifest) { log.Warnf("Please check for root manifest file, you may download one from the repository mirror, or try `tiup mirror set` to force reset it.") @@ -100,13 +106,30 @@ the latest stable version will be downloaded from the repository.`, return nil }, RunE: func(cmd *cobra.Command, args []string) error { - if printVersion && len(args) == 0 { - fmt.Println(version.NewTiUPVersion().String()) - return nil + if len(args) == 0 { + return cmd.Help() } env := environment.GlobalEnv() - if binary != "" { - component, ver := environment.ParseCompVersion(binary) + + // TBD: change this flag to subcommand + + // We assume the first unknown parameter is the component name and following + // parameters will be transparent passed because registered flags and subcommands + // will be parsed correctly. + // e.g: tiup --tag mytag --rm playground --db 3 --pd 3 --kv 4 + // => run "playground" with parameters "--db 3 --pd 3 --kv 4" + // tiup --tag mytag --binpath /xxx/tikv-server tikv + switch args[0] { + case "--help", "-h": + return cmd.Help() + case "--version", "-v": + fmt.Println(version.NewTiUPVersion().String()) + return nil + case "--binary": + if len(args) < 2 { + return fmt.Errorf("flag needs an argument: %s", args[0]) + } + component, ver := environment.ParseCompVersion(args[1]) selectedVer, err := env.SelectInstalledVersion(component, ver) if err != nil { return err @@ -117,36 +140,31 @@ the latest stable version will be downloaded from the repository.`, } fmt.Println(binaryPath) return nil - } - if len(args) > 0 { - // We assume the first unknown parameter is the component name and following - // parameters will be transparent passed because registered flags and subcommands - // will be parsed correctly. - // e.g: tiup --tag mytag --rm playground --db 3 --pd 3 --kv 4 - // => run "playground" with parameters "--db 3 --pd 3 --kv 4" - // tiup --tag mytag --binpath /xxx/tikv-server tikv - var transparentParams []string - componentSpec := args[0] - for i, arg := range os.Args { - if arg == componentSpec { - transparentParams = os.Args[i+1:] - break - } + case "--binpath": + if len(args) < 2 { + return fmt.Errorf("flag needs an argument: %s", args[0]) } - if len(transparentParams) > 0 && transparentParams[0] == "--" { - transparentParams = transparentParams[1:] + binPath = args[1] + args = args[2:] + case "--tag", "-T": + if len(args) < 2 { + return fmt.Errorf("flag needs an argument: %s", args[0]) } - - teleCommand = fmt.Sprintf("%s %s", cmd.CommandPath(), componentSpec) - return tiupexec.RunComponent(env, tag, componentSpec, binPath, transparentParams) + tag = args[1] + args = args[2:] } - return cmd.Help() - }, - PersistentPostRunE: func(cmd *cobra.Command, args []string) error { - if env := environment.GlobalEnv(); env != nil { - return env.Close() + if len(args) < 1 { + return cmd.Help() } - return nil + + componentSpec := args[0] + args = args[1:] + if len(args) > 0 && args[0] == "--" { + args = args[1:] + } + + teleCommand = fmt.Sprintf("%s %s", cmd.CommandPath(), componentSpec) + return tiupexec.RunComponent(env, tag, componentSpec, binPath, args) }, SilenceUsage: true, // implement auto completion for tiup components @@ -181,12 +199,12 @@ the latest stable version will be downloaded from the repository.`, }, } - rootCmd.PersistentFlags().BoolVarP(&repoOpts.SkipVersionCheck, "skip-version-check", "", false, "Skip the strict version check, by default a version must be a valid SemVer string") - rootCmd.Flags().StringVar(&binary, "binary", "", "Print binary path of a specific version of a component `[:version]`\n"+ + // useless, exist to generate help information + rootCmd.Flags().String("binary", "", "Print binary path of a specific version of a component `[:version]`\n"+ "and the latest version installed will be selected if no version specified") - rootCmd.Flags().StringVarP(&tag, "tag", "T", "", "[Deprecated] Specify a tag for component instance") - rootCmd.Flags().StringVar(&binPath, "binpath", "", "Specify the binary path of component instance") - rootCmd.Flags().BoolVarP(&printVersion, "version", "v", false, "Print the version of tiup") + rootCmd.Flags().StringP("tag", "T", "", "[Deprecated] Specify a tag for component instance") + rootCmd.Flags().String("binpath", "", "Specify the binary path of component instance") + rootCmd.Flags().BoolP("version", "v", false, "Print the version of tiup") rootCmd.AddCommand( newInstallCmd(), @@ -198,25 +216,8 @@ the latest stable version will be downloaded from the repository.`, newMirrorCmd(), newTelemetryCmd(), newEnvCmd(), + newHistoryCmd(), ) - - originHelpFunc := rootCmd.HelpFunc() - rootCmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { - cmd, _, _ = cmd.Root().Find(args) - if len(args) < 2 || cmd != rootCmd { - originHelpFunc(cmd, args) - return - } - - env, _ := environment.InitEnv(repoOpts) - environment.SetGlobalEnv(env) - _ = cmd.RunE(cmd, args) - }) - - rootCmd.SetHelpCommand(newHelpCmd()) - // If env is inited before, localdata.InitProfile() will return a valid profile - // or it will return an invalid one but still print usage - rootCmd.SetUsageTemplate(usageTemplate(localdata.InitProfile())) } // Execute parses the command line arguments and calls proper functions @@ -248,6 +249,12 @@ func Execute() { // us a dedicated package for that reportEnabled = false } else { + // record TiUP execution history + err := environment.HistoryRecord(env, os.Args, start, code) + if err != nil { + log.Warnf("Record TiUP execution history log failed: %v", err) + } + teleMeta, _, err := telemetry.GetMeta(env) if err == nil { reportEnabled = teleMeta.Status == telemetry.EnableStatus diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go index ca976c7e92..6bad82f4fc 100644 --- a/components/cluster/command/root.go +++ b/components/cluster/command/root.go @@ -120,7 +120,7 @@ func init() { env, err = tiupmeta.InitEnv(repository.Options{ GOOS: "linux", GOARCH: "amd64", - }) + }, repository.MirrorOptions{}) if err != nil { return err } diff --git a/components/dm/command/root.go b/components/dm/command/root.go index 4246c5088f..b6bf79ac9c 100644 --- a/components/dm/command/root.go +++ b/components/dm/command/root.go @@ -91,7 +91,7 @@ please backup your data before process.`, env, err = tiupmeta.InitEnv(repository.Options{ GOOS: "linux", GOARCH: "amd64", - }) + }, repository.MirrorOptions{}) if err != nil { return err } diff --git a/components/playground/main.go b/components/playground/main.go index a2c159733b..bdade14803 100644 --- a/components/playground/main.go +++ b/components/playground/main.go @@ -218,7 +218,7 @@ Examples: return err } - env, err := environment.InitEnv(repository.Options{}) + env, err := environment.InitEnv(repository.Options{}, repository.MirrorOptions{}) if err != nil { return err } diff --git a/embed/examples/cluster/topology.example.yaml b/embed/examples/cluster/topology.example.yaml index 0edac37932..c81f4824fe 100644 --- a/embed/examples/cluster/topology.example.yaml +++ b/embed/examples/cluster/topology.example.yaml @@ -327,3 +327,5 @@ alertmanager_servers: # data_dir: "/tidb-data/alertmanager-9093" # # Alertmanager log file storage directory. # log_dir: "/tidb-deploy/alertmanager-9093/log" + # # Alertmanager config file storage directory. + # config_file: "/tidb-deploy/alertmanager-9093/bin/alertmanager/alertmanager.yml" diff --git a/embed/templates/scripts/run_dm-master.sh.tpl b/embed/templates/scripts/run_dm-master.sh.tpl index 246e2097ad..39fde5e418 100644 --- a/embed/templates/scripts/run_dm-master.sh.tpl +++ b/embed/templates/scripts/run_dm-master.sh.tpl @@ -25,7 +25,7 @@ exec bin/dm-master/dm-master \ --v1-sources-path="{{.V1SourcePath}}" \ {{- end}} --name="{{.Name}}" \ - --master-addr="{{.IP}}:{{.Port}}" \ + --master-addr="0.0.0.0:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ diff --git a/embed/templates/scripts/run_dm-master_scale.sh.tpl b/embed/templates/scripts/run_dm-master_scale.sh.tpl index 90572b0241..13a31977aa 100644 --- a/embed/templates/scripts/run_dm-master_scale.sh.tpl +++ b/embed/templates/scripts/run_dm-master_scale.sh.tpl @@ -22,7 +22,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-master/d exec bin/dm-master/dm-master \ {{- end}} --name="{{.Name}}" \ - --master-addr="{{.IP}}:{{.Port}}" \ + --master-addr="0.0.0.0:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ diff --git a/embed/templates/scripts/run_dm-worker.sh.tpl b/embed/templates/scripts/run_dm-worker.sh.tpl index df11e96e20..f1074b4967 100644 --- a/embed/templates/scripts/run_dm-worker.sh.tpl +++ b/embed/templates/scripts/run_dm-worker.sh.tpl @@ -23,7 +23,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-worker/d exec bin/dm-worker/dm-worker \ {{- end}} --name="{{.Name}}" \ - --worker-addr="{{.IP}}:{{.Port}}" \ + --worker-addr="0.0.0.0:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --log-file="{{.LogDir}}/dm-worker.log" \ --join="{{template "MasterList" .Endpoints}}" \ diff --git a/go.mod b/go.mod index 4a78de0974..3653b1c439 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20220125073028-58f2ac94aa38 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect github.com/pingcap/tidb-insight/collector v0.0.0-20220111101533-227008e9835b + github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.32.1 github.com/prometheus/prom2json v1.3.0 diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index c07c3a3b1e..b92a62200c 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -600,6 +600,7 @@ func DestroyClusterTombstone( if !tombstone { pumpServers = append(pumpServers, s) + continue } nodes = append(nodes, id) @@ -629,6 +630,7 @@ func DestroyClusterTombstone( if !tombstone { drainerServers = append(drainerServers, s) + continue } nodes = append(nodes, id) diff --git a/pkg/environment/env.go b/pkg/environment/env.go index a2d35f69bf..4bb55dd368 100644 --- a/pkg/environment/env.go +++ b/pkg/environment/env.go @@ -81,7 +81,7 @@ type Environment struct { } // InitEnv creates a new Environment object configured using env vars and defaults. -func InitEnv(options repository.Options) (*Environment, error) { +func InitEnv(options repository.Options, mOpt repository.MirrorOptions) (*Environment, error) { if env := GlobalEnv(); env != nil { return env, nil } @@ -92,7 +92,7 @@ func InitEnv(options repository.Options) (*Environment, error) { // Initialize the repository // Replace the mirror if some sub-commands use different mirror address mirrorAddr := Mirror() - mirror := repository.NewMirror(mirrorAddr, repository.MirrorOptions{}) + mirror := repository.NewMirror(mirrorAddr, mOpt) if err := mirror.Open(); err != nil { return nil, err } diff --git a/pkg/environment/history.go b/pkg/environment/history.go new file mode 100644 index 0000000000..a71e303fb7 --- /dev/null +++ b/pkg/environment/history.go @@ -0,0 +1,244 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + "github.com/pingcap/tiup/pkg/tui" + "github.com/pingcap/tiup/pkg/utils" + "github.com/pkg/errors" +) + +const ( + // HistoryDir history save path + HistoryDir = "history" + historyPrefix = "tiup-history-" + historySize int64 = 1024 * 64 // history file default size is 64k +) + +// commandRow type of command history row +type historyRow struct { + Date time.Time `json:"time"` + Command string `json:"command"` + Code int `json:"exit_code"` +} + +// historyItem record history row file item +type historyItem struct { + path string + info fs.FileInfo + index int +} + +// HistoryRecord record tiup exec cmd +func HistoryRecord(env *Environment, command []string, date time.Time, code int) error { + if env == nil { + return nil + } + + historyPath := env.LocalPath(HistoryDir) + if utils.IsNotExist(historyPath) { + err := os.MkdirAll(historyPath, 0755) + if err != nil { + return err + } + } + + h := &historyRow{ + Command: strings.Join(command, " "), + Date: date, + Code: code, + } + + return h.save(historyPath) +} + +// save save commandRow to file +func (r *historyRow) save(dir string) error { + rBytes, err := json.Marshal(r) + if err != nil { + return err + } + + historyFile := getLatestHistoryFile(dir) + + f, err := os.OpenFile(historyFile.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(append(rBytes, []byte("\n")...)) + return err +} + +// GetHistory get tiup history +func (env *Environment) GetHistory(count int, all bool) ([]*historyRow, error) { + fList, err := getHistoryFileList(env.LocalPath(HistoryDir)) + if err != nil { + return nil, err + } + rows := []*historyRow{} + for _, f := range fList { + rs, err := f.getHistory() + if err != nil { + return rows, err + } + if (len(rows)+len(rs)) > count && !all { + i := len(rows) + len(rs) - count + rows = append(rs[i:], rows...) + break + } + + rows = append(rs, rows...) + } + return rows, nil +} + +// DeleteHistory delete history file +func (env *Environment) DeleteHistory(retainDays int, skipConfirm bool) error { + if retainDays < 0 { + return errors.Errorf("retainDays cannot be less than 0") + } + + // history file before `DelBeforeTime` will be deleted + oneDayDuration, _ := time.ParseDuration("-24h") + delBeforeTime := time.Now().Add(oneDayDuration * time.Duration(retainDays)) + + if !skipConfirm { + fmt.Printf("History logs before %s will be %s!\n", + color.HiYellowString(delBeforeTime.Format("2006-01-02T15:04:05")), + color.HiYellowString("deleted"), + ) + if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { + return err + } + } + + fList, err := getHistoryFileList(env.LocalPath(HistoryDir)) + if err != nil { + return err + } + + if len(fList) == 0 { + return nil + } + + for _, f := range fList { + if f.info.ModTime().Before(delBeforeTime) { + err := os.Remove(f.path) + if err != nil { + return err + } + continue + } + } + return nil +} + +// getHistory get tiup history execution row +func (i *historyItem) getHistory() ([]*historyRow, error) { + rows := []*historyRow{} + + fi, err := os.Open(i.path) + if err != nil { + return rows, err + } + defer fi.Close() + + br := bufio.NewReader(fi) + for { + a, _, c := br.ReadLine() + if c == io.EOF { + break + } + r := &historyRow{} + // ignore + err := json.Unmarshal(a, r) + if err != nil { + continue + } + rows = append(rows, r) + } + + return rows, nil +} + +// getHistoryFileList get the history file list +func getHistoryFileList(dir string) ([]historyItem, error) { + fileInfos, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + hfileList := []historyItem{} + for _, fi := range fileInfos { + if fi.IsDir() { + continue + } + + // another suffix + // ex: tiup-history-0.bak + i, err := strconv.Atoi((strings.TrimPrefix(fi.Name(), historyPrefix))) + if err != nil { + continue + } + + fInfo, _ := fi.Info() + hfileList = append(hfileList, historyItem{ + path: filepath.Join(dir, fi.Name()), + index: i, + info: fInfo, + }) + } + + sort.Slice(hfileList, func(i, j int) bool { + return hfileList[i].index > hfileList[j].index + }) + + return hfileList, nil +} + +// getLatestHistoryFile get the latest history file, use index 0 if it doesn't exist +func getLatestHistoryFile(dir string) (item historyItem) { + fileList, err := getHistoryFileList(dir) + // start from 0 + if len(fileList) == 0 || err != nil { + item.index = 0 + item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) + return + } + + latestItem := fileList[0] + + if latestItem.info.Size() >= historySize { + item.index = latestItem.index + 1 + item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) + } else { + item = latestItem + } + + return +} diff --git a/pkg/repository/model/model.go b/pkg/repository/model/model.go index 5024cdc74c..6352adc302 100644 --- a/pkg/repository/model/model.go +++ b/pkg/repository/model/model.go @@ -134,10 +134,22 @@ func (m *model) Rotate(manifest *v1manifest.Manifest) error { return err } + // write new 'root.json' file with verion prefix manifestFilename := fmt.Sprintf("%d.root.json", root.Version) if err := m.txn.WriteManifest(manifestFilename, manifest); err != nil { return err } + /* not yet update the 'root.json' without version prefix, as we don't + * have a '1.root.json', so the 'root.json' is playing the role of initial + * '1.root.json', clients are updating to the latest 'n.root.json' no + * matter older ones are expired or not + * maybe we could update the 'root.json' some day when we have many many + * versions of root.json available and the updating process from old clients + * are causing performance issues + */ + // if err := m.txn.WriteManifest("root.json", manifest); err != nil { + // return err + // } fi, err := m.txn.Stat(manifestFilename) if err != nil { diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index 8924ace8ef..0dd38fb3e1 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -22,7 +22,6 @@ type Repository struct { // Options represents options for a repository type Options struct { - SkipVersionCheck bool GOOS string GOARCH string DisableDecompress bool