diff --git a/.github/workflows/integrate-cluster-cmd.yaml b/.github/workflows/integrate-cluster-cmd.yaml index db14720d69..9bac1e5702 100644 --- a/.github/workflows/integrate-cluster-cmd.yaml +++ b/.github/workflows/integrate-cluster-cmd.yaml @@ -84,6 +84,7 @@ jobs: - name: Collect component log working-directory: ${{ env.working-directory }} + if: ${{ failure() }} # if: always() run: | docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs @@ -91,6 +92,7 @@ jobs: - name: Detect error log working-directory: ${{ env.working-directory }} + if: ${{ failure() }} # if: always() run: | bash ./tests/tiup-cluster/script/detect_error.sh ./logs/ @@ -105,7 +107,8 @@ jobs: - name: Output cluster debug log working-directory: ${{ env.working-directory }} - if: always() + if: ${{ failure() }} + # if: always() run: | pwd docker ps diff --git a/.github/workflows/integrate-cluster-scale.yaml b/.github/workflows/integrate-cluster-scale.yaml index 7ce19a5f37..78d4b76580 100644 --- a/.github/workflows/integrate-cluster-scale.yaml +++ b/.github/workflows/integrate-cluster-scale.yaml @@ -84,6 +84,7 @@ jobs: - name: Collect component log working-directory: ${{ env.working-directory }} + if: ${{ failure() }} # if: always() run: | docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs @@ -91,6 +92,7 @@ jobs: - name: Detect error log working-directory: ${{ env.working-directory }} + if: ${{ failure() }} # if: always() run: | bash ./tests/tiup-cluster/script/detect_error.sh ./logs/ diff --git a/.github/workflows/integrate-dm.yaml b/.github/workflows/integrate-dm.yaml index 37ef609075..cb18ae3387 100644 --- a/.github/workflows/integrate-dm.yaml +++ b/.github/workflows/integrate-dm.yaml @@ -67,7 +67,7 @@ jobs: # with --dev the first run will fail for unknow reason, just retry it and will success now.. run: | cd ${{ env.working-directory }}/docker - TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./up.sh --daemon --dev --compose ./docker-compose.dm.yml || TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./up.sh --daemon --dev --compose ./docker-compose.dm.yml + TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./up.sh --daemon --dev --compose ./docker-compose.dm.yml - name: Check running containers run: | @@ -87,8 +87,8 @@ jobs: - name: Collect component log working-directory: ${{ env.working-directory }} + if: ${{ failure() }} # if: ${{ failure() }} - if: always() run: | docker exec tiup-cluster-control bash -c 'mkdir -p /tiup-cluster/logs; [[ -d ~/.tiup/logs ]] && find ~/.tiup/logs -type f -name "*.log" -exec cp {} /tiup-cluster/logs \; || true' ls ${{ env.working-directory }} @@ -97,7 +97,7 @@ jobs: - name: Upload component log if: ${{ failure() }} # if: always() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: dm_logs path: ${{ env.working-directory }}/dm.logs.tar.gz diff --git a/.github/workflows/integrate-playground.yaml b/.github/workflows/integrate-playground.yaml index c8c407f0d7..48d075c22c 100644 --- a/.github/workflows/integrate-playground.yaml +++ b/.github/workflows/integrate-playground.yaml @@ -77,7 +77,7 @@ jobs: - name: Upload component log if: ${{ failure() }} # if: always() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: playground_logs path: ${{ env.working-directory }}/playground.logs.tar.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index f1da081361..9ac4ccc57e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ TiUP Changelog +## [1.16.1] 2024-10-31 + +### New Features + +- Deploy DM in `tiup-playground` (#2465, @GMHDBJD, @siddontang) + +### Fixes + +- Fix several racings during download, do not upgrade nightly automatically in `tiup-cluster` (#2458, @xhebox) +- Fix possible panic when `tiup-playground` failed to start (#2457, @xhebox) +- Respect `component_versions` when `tiup-cluster` scales (#2451, @djshow832) +- Code clean: replace 'math/rand' with 'crypto/rand' (#2455, @bb7133) +- Fix tiup cannot update itself when tiup comonent exist (#2443, @nexustar) +- Do not check HTTP port for TiFlash 7.1.0 or above (#2440, @Lloyd-Pottiger) +- Also hide other password args (#2436, @xhebox) + +### Improvements + +- Avoid unnecessary primary transfer for pdms mode in `tiup-cluster` (#2414, @HuSharp) +- Add `--port-offset` for `tiup-playground` to start multiple instances without port conflicts (#2453, @breezewish) +- Start with a name for pdms mode in `tiup-cluster` and `tiup-playground` (#2438, #2446, #2447, @HuSharp) +- Remove prometheus systemd and interrupts collector (#2445, @xhebox) +- Add example of setting the tiflash-proxy topology (#2444, @JaySon-Huang) +- Improve cluster restart messaging for `tiup-cluster` (#2442, @zph) +- Print version when filling `tiup-playground --xx.binpath` (#2334, @HuSharp) +- Only display `--comments` when needed for `tiup-playground` (#2314, @dveeden) + ## [1.16.0] 2024-06-27 ### New Features diff --git a/cmd/mirror.go b/cmd/mirror.go index d09aaaac67..9fdb6b3d9c 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -163,8 +163,9 @@ func newMirrorShowCmd() *cobra.Command { // the `mirror set` sub command func newMirrorSetCmd() *cobra.Command { var ( - root string - reset bool + root string + reset bool + silent bool ) cmd := &cobra.Command{ Use: "set ", @@ -198,12 +199,15 @@ The root manifest in $TIUP_HOME will be replaced with the one in given repositor log.Errorf("Failed to set mirror: %s\n", err.Error()) return err } - fmt.Printf("Successfully set mirror to %s\n", addr) + if !silent { + fmt.Printf("Successfully set mirror to %s\n", addr) + } return nil }, } cmd.Flags().StringVarP(&root, "root", "r", root, "Specify the path of `root.json`") cmd.Flags().BoolVar(&reset, "reset", false, "Reset mirror to use the default address.") + cmd.Flags().BoolVar(&silent, "silent", false, "Skip non-warning messages.") return cmd } @@ -955,7 +959,7 @@ func newMirrorCloneCmd() *cobra.Command { cmd := &cobra.Command{ Use: "clone [global version]", Example: ` tiup mirror clone /path/to/local --arch amd64,arm64 --os linux,darwin # Specify the architectures and OSs - tiup mirror clone /path/to/local --os linux v6.1.0 v5.4.0 # Specify multiple versions + tiup mirror clone /path/to/local --os linux v6.1.0 v5.4.0 # Specify multiple versions tiup mirror clone /path/to/local --full # Build a full local mirror tiup mirror clone /path/to/local --tikv v4 --prefix # Specify the version via prefix tiup mirror clone /path/to/local --tidb all --pd all # Download all version for specific component`, diff --git a/components/playground/command.go b/components/playground/command.go index 36cdccd65b..62a1b9232b 100644 --- a/components/playground/command.go +++ b/components/playground/command.go @@ -61,6 +61,8 @@ func buildCommands(tp CommandType, opt *BootOptions) (cmds []Command) { {"ticdc", opt.TiCDC}, {"tikv-cdc", opt.TiKVCDC}, {"drainer", opt.Drainer}, + {"dm-master", opt.DMMaster}, + {"dm-worker", opt.DMWorker}, } for _, cmd := range commands { @@ -113,6 +115,8 @@ func newScaleOut() *cobra.Command { cmd.Flags().StringVarP(&opt.TSO.Host, "tso.host", "", opt.TSO.Host, "Playground TSO host. If not provided, TSO will still use `host` flag as its host") cmd.Flags().StringVarP(&opt.Scheduling.Host, "scheduling.host", "", opt.Scheduling.Host, "Playground Scheduling host. If not provided, Scheduling will still use `host` flag as its host") cmd.Flags().StringVarP(&opt.TiProxy.Host, "tiproxy.host", "", opt.PD.Host, "Playground TiProxy host. If not provided, TiProxy will still use `host` flag as its host") + cmd.Flags().IntVarP(&opt.DMMaster.Num, "dm-master", "", opt.DMMaster.Num, "DM-master instance number") + cmd.Flags().IntVarP(&opt.DMWorker.Num, "dm-worker", "", opt.DMWorker.Num, "DM-worker instance number") cmd.Flags().StringVarP(&opt.TiDB.ConfigPath, "db.config", "", opt.TiDB.ConfigPath, "TiDB instance configuration file") cmd.Flags().StringVarP(&opt.TiKV.ConfigPath, "kv.config", "", opt.TiKV.ConfigPath, "TiKV instance configuration file") @@ -123,6 +127,8 @@ func newScaleOut() *cobra.Command { cmd.Flags().StringVarP(&opt.TiProxy.ConfigPath, "tiproxy.config", "", opt.TiProxy.ConfigPath, "TiProxy instance configuration file") cmd.Flags().StringVarP(&opt.Pump.ConfigPath, "pump.config", "", opt.Pump.ConfigPath, "Pump instance configuration file") cmd.Flags().StringVarP(&opt.Drainer.ConfigPath, "drainer.config", "", opt.Drainer.ConfigPath, "Drainer instance configuration file") + cmd.Flags().StringVarP(&opt.DMMaster.ConfigPath, "dm-master.config", "", opt.DMMaster.ConfigPath, "DM-master instance configuration file") + cmd.Flags().StringVarP(&opt.DMWorker.ConfigPath, "dm-worker.config", "", opt.DMWorker.ConfigPath, "DM-worker instance configuration file") cmd.Flags().StringVarP(&opt.TiDB.BinPath, "db.binpath", "", opt.TiDB.BinPath, "TiDB instance binary path") cmd.Flags().StringVarP(&opt.TiKV.BinPath, "kv.binpath", "", opt.TiKV.BinPath, "TiKV instance binary path") @@ -135,6 +141,8 @@ func newScaleOut() *cobra.Command { cmd.Flags().StringVarP(&opt.TiKVCDC.BinPath, "kvcdc.binpath", "", opt.TiKVCDC.BinPath, "TiKVCDC instance binary path") cmd.Flags().StringVarP(&opt.Pump.BinPath, "pump.binpath", "", opt.Pump.BinPath, "Pump instance binary path") cmd.Flags().StringVarP(&opt.Drainer.BinPath, "drainer.binpath", "", opt.Drainer.BinPath, "Drainer instance binary path") + cmd.Flags().StringVarP(&opt.DMMaster.BinPath, "dm-master.binpath", "", opt.DMMaster.BinPath, "DM-master instance binary path") + cmd.Flags().StringVarP(&opt.DMWorker.BinPath, "dm-worker.binpath", "", opt.DMWorker.BinPath, "DM-worker instance binary path") return cmd } diff --git a/components/playground/grafana.go b/components/playground/grafana.go index bccae98c76..1a5573f3f7 100644 --- a/components/playground/grafana.go +++ b/components/playground/grafana.go @@ -151,12 +151,8 @@ var clusterName = "Test-Cluster" // dir should contains files untar the grafana. // return not error iff the Cmd is started successfully. -func (g *grafana) start(ctx context.Context, dir string, p8sURL string) (err error) { - g.port, err = utils.GetFreePort(g.host, g.port) - if err != nil { - return err - } - +func (g *grafana) start(ctx context.Context, dir string, portOffset int, p8sURL string) (err error) { + g.port = utils.MustGetFreePort(g.host, g.port, portOffset) fname := filepath.Join(dir, "conf", "provisioning", "dashboards", "dashboard.yml") err = writeDashboardConfig(fname, clusterName, filepath.Join(dir, "dashboards")) if err != nil { diff --git a/components/playground/instance/dm_master.go b/components/playground/instance/dm_master.go new file mode 100644 index 0000000000..e621458b81 --- /dev/null +++ b/components/playground/instance/dm_master.go @@ -0,0 +1,90 @@ +package instance + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/pingcap/tiup/pkg/utils" +) + +// DMMaster represent a DM master instance. +type DMMaster struct { + instance + Process + initEndpoints []*DMMaster +} + +var _ Instance = &DMMaster{} + +// NewDMMaster create a new DMMaster instance. +func NewDMMaster(binPath string, dir, host, configPath string, portOffset int, id int, port int) *DMMaster { + if port <= 0 { + port = 8261 + } + return &DMMaster{ + instance: instance{ + BinPath: binPath, + ID: id, + Dir: dir, + Host: host, + Port: utils.MustGetFreePort(host, 8291, portOffset), + // Similar like PD's client port, here use StatusPort for Master Port. + StatusPort: utils.MustGetFreePort(host, port, portOffset), + ConfigPath: configPath, + }, + } +} + +// Name return the name of the instance. +func (m *DMMaster) Name() string { + return fmt.Sprintf("dm-master-%d", m.ID) +} + +// Start starts the instance. +func (m *DMMaster) Start(ctx context.Context) error { + args := []string{ + fmt.Sprintf("--name=%s", m.Name()), + fmt.Sprintf("--master-addr=http://%s", utils.JoinHostPort(m.Host, m.StatusPort)), + fmt.Sprintf("--advertise-addr=http://%s", utils.JoinHostPort(AdvertiseHost(m.Host), m.StatusPort)), + fmt.Sprintf("--peer-urls=http://%s", utils.JoinHostPort(m.Host, m.Port)), + fmt.Sprintf("--advertise-peer-urls=http://%s", utils.JoinHostPort(AdvertiseHost(m.Host), m.Port)), + fmt.Sprintf("--log-file=%s", m.LogFile()), + } + + endpoints := make([]string, 0) + for _, master := range m.initEndpoints { + endpoints = append(endpoints, fmt.Sprintf("%s=http://%s", master.Name(), utils.JoinHostPort(master.Host, master.Port))) + } + args = append(args, fmt.Sprintf("--initial-cluster=%s", strings.Join(endpoints, ","))) + + if m.ConfigPath != "" { + args = append(args, fmt.Sprintf("--config=%s", m.ConfigPath)) + } + + m.Process = &process{cmd: PrepareCommand(ctx, m.BinPath, args, nil, m.Dir)} + + logIfErr(m.Process.SetOutputFile(m.LogFile())) + return m.Process.Start() +} + +// SetInitEndpoints set the initial endpoints for the DM master. +func (m *DMMaster) SetInitEndpoints(endpoints []*DMMaster) { + m.initEndpoints = endpoints +} + +// Component return the component of the instance. +func (m *DMMaster) Component() string { + return "dm-master" +} + +// LogFile return the log file path of the instance. +func (m *DMMaster) LogFile() string { + return filepath.Join(m.Dir, "dm-master.log") +} + +// Addr return the address of the instance. +func (m *DMMaster) Addr() string { + return utils.JoinHostPort(m.Host, m.StatusPort) +} diff --git a/components/playground/instance/dm_worker.go b/components/playground/instance/dm_worker.go new file mode 100644 index 0000000000..61c5e748f1 --- /dev/null +++ b/components/playground/instance/dm_worker.go @@ -0,0 +1,83 @@ +package instance + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/pingcap/tiup/pkg/utils" +) + +// DMWorker represent a DM worker instance. +type DMWorker struct { + instance + Process + + masters []*DMMaster +} + +var _ Instance = &DMWorker{} + +// NewDMWorker create a DMWorker instance. +func NewDMWorker(binPath string, dir, host, configPath string, portOffset int, id int, port int, masters []*DMMaster) *DMWorker { + if port <= 0 { + port = 8262 + } + return &DMWorker{ + instance: instance{ + BinPath: binPath, + ID: id, + Dir: dir, + Host: host, + Port: utils.MustGetFreePort(host, port, portOffset), + ConfigPath: configPath, + }, + masters: masters, + } +} + +// MasterAddrs return the master addresses. +func (w *DMWorker) MasterAddrs() []string { + var addrs []string + for _, master := range w.masters { + addrs = append(addrs, utils.JoinHostPort(AdvertiseHost(master.Host), master.StatusPort)) + } + return addrs +} + +// Name return the name of the instance. +func (w *DMWorker) Name() string { + return fmt.Sprintf("dm-worker-%d", w.ID) +} + +// Start starts the instance. +func (w *DMWorker) Start(ctx context.Context) error { + args := []string{ + fmt.Sprintf("--name=%s", w.Name()), + fmt.Sprintf("--worker-addr=%s", utils.JoinHostPort(w.Host, w.Port)), + fmt.Sprintf("--advertise-addr=%s", utils.JoinHostPort(AdvertiseHost(w.Host), w.Port)), + fmt.Sprintf("--join=%s", strings.Join(w.MasterAddrs(), ",")), + fmt.Sprintf("--log-file=%s", w.LogFile()), + } + + if w.ConfigPath != "" { + args = append(args, fmt.Sprintf("--config=%s", w.ConfigPath)) + } + + w.Process = &process{cmd: PrepareCommand(ctx, w.BinPath, args, nil, w.Dir)} + + logIfErr(w.Process.SetOutputFile(w.LogFile())) + + return w.Process.Start() +} + +// Component return the component of the instance. +func (w *DMWorker) Component() string { + return "dm-worker" +} + +// LogFile return the log file of the instance. +func (w *DMWorker) LogFile() string { + return filepath.Join(w.Dir, "dm-worker.log") +} diff --git a/components/playground/instance/drainer.go b/components/playground/instance/drainer.go index f72aa4523f..30bf937871 100644 --- a/components/playground/instance/drainer.go +++ b/components/playground/instance/drainer.go @@ -32,14 +32,14 @@ type Drainer struct { var _ Instance = &Drainer{} // NewDrainer create a Drainer instance. -func NewDrainer(binPath string, dir, host, configPath string, id int, pds []*PDInstance) *Drainer { +func NewDrainer(binPath string, dir, host, configPath string, portOffset int, id int, pds []*PDInstance) *Drainer { d := &Drainer{ instance: instance{ BinPath: binPath, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, 8250), + Port: utils.MustGetFreePort(host, 8250, portOffset), ConfigPath: configPath, }, pds: pds, diff --git a/components/playground/instance/instance.go b/components/playground/instance/instance.go index 9766fd67f2..78da752c33 100644 --- a/components/playground/instance/instance.go +++ b/components/playground/instance/instance.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/spec" tiupexec "github.com/pingcap/tiup/pkg/exec" + "github.com/pingcap/tiup/pkg/tui/colorstr" "github.com/pingcap/tiup/pkg/utils" ) @@ -81,7 +82,7 @@ type Instance interface { // The implementation should be safe to call Wait multi times. Wait() error // PrepareBinary use given binpath or download from tiup mirrors. - PrepareBinary(componentName string, version utils.Version) error + PrepareBinary(binaryName string, componentName string, version utils.Version) error } func (inst *instance) MetricAddr() (r MetricAddr) { @@ -91,16 +92,16 @@ func (inst *instance) MetricAddr() (r MetricAddr) { return } -func (inst *instance) PrepareBinary(componentName string, version utils.Version) error { - instanceBinPath, err := tiupexec.PrepareBinary(componentName, version, inst.BinPath) +func (inst *instance) PrepareBinary(binaryName string, componentName string, version utils.Version) error { + instanceBinPath, err := tiupexec.PrepareBinary(binaryName, version, inst.BinPath) if err != nil { return err } // distinguish whether the instance is started by specific binary path. if inst.BinPath == "" { - fmt.Printf("Start %s instance:%s\n", componentName, version) + colorstr.Printf("[dark_gray]Start %s instance: %s[reset]\n", componentName, version) } else { - fmt.Printf("Start %s instance:%s\n", componentName, instanceBinPath) + colorstr.Printf("[dark_gray]Start %s instance: %s[reset]\n", componentName, instanceBinPath) } inst.Version = version inst.BinPath = instanceBinPath diff --git a/components/playground/instance/pd.go b/components/playground/instance/pd.go index 3c1d6f8636..8c699b990c 100644 --- a/components/playground/instance/pd.go +++ b/components/playground/instance/pd.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/utils" ) @@ -49,7 +50,7 @@ type PDInstance struct { } // NewPDInstance return a PDInstance -func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int, isCSEMode bool) *PDInstance { +func NewPDInstance(role PDRole, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, port int, isCSEMode bool) *PDInstance { if port <= 0 { port = 2379 } @@ -59,8 +60,8 @@ func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, p ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, 2380), - StatusPort: utils.MustGetFreePort(host, port), + Port: utils.MustGetFreePort(host, 2380, portOffset), + StatusPort: utils.MustGetFreePort(host, port, portOffset), ConfigPath: configPath, }, Role: role, @@ -83,7 +84,14 @@ func (inst *PDInstance) InitCluster(pds []*PDInstance) *PDInstance { // Name return the name of pd. func (inst *PDInstance) Name() string { - return fmt.Sprintf("pd-%d", inst.ID) + switch inst.Role { + case PDRoleTSO: + return fmt.Sprintf("tso-%d", inst.ID) + case PDRoleScheduling: + return fmt.Sprintf("scheduling-%d", inst.ID) + default: + return fmt.Sprintf("pd-%d", inst.ID) + } } // Start calls set inst.cmd and Start @@ -142,6 +150,9 @@ func (inst *PDInstance) Start(ctx context.Context) error { fmt.Sprintf("--log-file=%s", inst.LogFile()), fmt.Sprintf("--config=%s", configPath), } + if tidbver.PDSupportMicroServicesWithName(inst.Version.String()) { + args = append(args, fmt.Sprintf("--name=%s", uid)) + } case PDRoleScheduling: endpoints := pdEndpoints(inst.pds, true) args = []string{ @@ -153,6 +164,9 @@ func (inst *PDInstance) Start(ctx context.Context) error { fmt.Sprintf("--log-file=%s", inst.LogFile()), fmt.Sprintf("--config=%s", configPath), } + if tidbver.PDSupportMicroServicesWithName(inst.Version.String()) { + args = append(args, fmt.Sprintf("--name=%s", uid)) + } } inst.Process = &process{cmd: PrepareCommand(ctx, inst.BinPath, args, nil, inst.Dir)} diff --git a/components/playground/instance/pd_config.go b/components/playground/instance/pd_config.go index d951299606..60091a3b0c 100644 --- a/components/playground/instance/pd_config.go +++ b/components/playground/instance/pd_config.go @@ -22,7 +22,7 @@ func (inst *PDInstance) getConfig() map[string]any { config["replication.enable-placement-rules"] = true config["replication.max-replica"] = 1 config["schedule.merge-schedule-limit"] = 0 - config["schedule.low-space-ration"] = 1.0 + config["schedule.low-space-ratio"] = 1.0 config["schedule.replica-schedule-limit"] = 500 } diff --git a/components/playground/instance/process.go b/components/playground/instance/process.go index 83ff1dcef9..b4468d9e59 100644 --- a/components/playground/instance/process.go +++ b/components/playground/instance/process.go @@ -11,6 +11,10 @@ import ( "github.com/pingcap/errors" ) +var ( + errNotUp = errors.New("not up") +) + // Process represent process to be run by playground type Process interface { Start() error @@ -32,6 +36,10 @@ type process struct { // Start the process func (p *process) Start() error { + if p == nil { + return errNotUp + } + // fmt.Printf("Starting `%s`: %s", filepath.Base(p.cmd.Path), strings.Join(p.cmd.Args, " ")) p.startTime = time.Now() return p.cmd.Start() @@ -39,6 +47,10 @@ func (p *process) Start() error { // Wait implements Instance interface. func (p *process) Wait() error { + if p == nil { + return errNotUp + } + p.waitOnce.Do(func() { p.waitErr = p.cmd.Wait() }) @@ -48,11 +60,18 @@ func (p *process) Wait() error { // Pid implements Instance interface. func (p *process) Pid() int { + if p == nil { + return 0 + } return p.cmd.Process.Pid } // Uptime implements Instance interface. func (p *process) Uptime() string { + if p == nil { + return errNotUp.Error() + } + s := p.cmd.ProcessState if s != nil { @@ -64,6 +83,10 @@ func (p *process) Uptime() string { } func (p *process) SetOutputFile(fname string) error { + if p == nil { + return errNotUp + } + f, err := os.OpenFile(fname, os.O_RDWR|os.O_CREATE, 0666) if err != nil { return errors.AddStack(err) @@ -73,11 +96,19 @@ func (p *process) SetOutputFile(fname string) error { } func (p *process) setOutput(w io.Writer) { + if p == nil { + return + } + p.cmd.Stdout = w p.cmd.Stderr = w } func (p *process) Cmd() *exec.Cmd { + if p == nil { + panic(errNotUp) + } + return p.cmd } diff --git a/components/playground/instance/pump.go b/components/playground/instance/pump.go index e4ca34b9cd..a5880fb7b4 100644 --- a/components/playground/instance/pump.go +++ b/components/playground/instance/pump.go @@ -34,14 +34,14 @@ type Pump struct { var _ Instance = &Pump{} // NewPump create a Pump instance. -func NewPump(binPath string, dir, host, configPath string, id int, pds []*PDInstance) *Pump { +func NewPump(binPath string, dir, host, configPath string, portOffset int, id int, pds []*PDInstance) *Pump { pump := &Pump{ instance: instance{ BinPath: binPath, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, 8249), + Port: utils.MustGetFreePort(host, 8249, portOffset), ConfigPath: configPath, }, pds: pds, diff --git a/components/playground/instance/ticdc.go b/components/playground/instance/ticdc.go index b233e8622f..62288f3525 100644 --- a/components/playground/instance/ticdc.go +++ b/components/playground/instance/ticdc.go @@ -33,7 +33,7 @@ type TiCDC struct { var _ Instance = &TiCDC{} // NewTiCDC create a TiCDC instance. -func NewTiCDC(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiCDC { +func NewTiCDC(binPath string, dir, host, configPath string, portOffset int, id int, port int, pds []*PDInstance) *TiCDC { if port <= 0 { port = 8300 } @@ -43,7 +43,7 @@ func NewTiCDC(binPath string, dir, host, configPath string, id int, port int, pd ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, port), + Port: utils.MustGetFreePort(host, port, portOffset), ConfigPath: configPath, }, pds: pds, diff --git a/components/playground/instance/tidb.go b/components/playground/instance/tidb.go index 20b89e92a2..b5496ffca1 100644 --- a/components/playground/instance/tidb.go +++ b/components/playground/instance/tidb.go @@ -34,7 +34,7 @@ type TiDBInstance struct { } // NewTiDBInstance return a TiDBInstance -func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isCSEMode bool) *TiDBInstance { +func NewTiDBInstance(binPath string, dir, host, configPath string, portOffset int, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isCSEMode bool) *TiDBInstance { if port <= 0 { port = 4000 } @@ -44,8 +44,8 @@ func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, port), - StatusPort: utils.MustGetFreePort("0.0.0.0", 10080), + Port: utils.MustGetFreePort(host, port, portOffset), + StatusPort: utils.MustGetFreePort("0.0.0.0", 10080, portOffset), ConfigPath: configPath, }, tiproxyCertDir: tiproxyCertDir, diff --git a/components/playground/instance/tiflash.go b/components/playground/instance/tiflash.go index 995c7df3b1..6a35d8a3d9 100644 --- a/components/playground/instance/tiflash.go +++ b/components/playground/instance/tiflash.go @@ -53,14 +53,14 @@ type TiFlashInstance struct { } // NewTiFlashInstance return a TiFlashInstance -func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance { +func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance { if role != TiFlashRoleNormal && role != TiFlashRoleDisaggWrite && role != TiFlashRoleDisaggCompute { panic(fmt.Sprintf("Unknown TiFlash role %s", role)) } httpPort := 8123 if !tidbver.TiFlashNotNeedHTTPPortConfig(version) { - httpPort = utils.MustGetFreePort(host, httpPort) + httpPort = utils.MustGetFreePort(host, httpPort, portOffset) } return &TiFlashInstance{ instance: instance{ @@ -69,15 +69,15 @@ func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, h Dir: dir, Host: host, Port: httpPort, - StatusPort: utils.MustGetFreePort(host, 8234), + StatusPort: utils.MustGetFreePort(host, 8234, portOffset), ConfigPath: configPath, }, Role: role, cseOpts: cseOptions, - TCPPort: utils.MustGetFreePort(host, 9100), // 9000 for default object store port - ServicePort: utils.MustGetFreePort(host, 3930), - ProxyPort: utils.MustGetFreePort(host, 20170), - ProxyStatusPort: utils.MustGetFreePort(host, 20292), + TCPPort: utils.MustGetFreePort(host, 9100, portOffset), // 9000 for default object store port + ServicePort: utils.MustGetFreePort(host, 3930, portOffset), + ProxyPort: utils.MustGetFreePort(host, 20170, portOffset), + ProxyStatusPort: utils.MustGetFreePort(host, 20292, portOffset), pds: pds, dbs: dbs, } diff --git a/components/playground/instance/tikv.go b/components/playground/instance/tikv.go index f70ec1bd96..8c97acd018 100644 --- a/components/playground/instance/tikv.go +++ b/components/playground/instance/tikv.go @@ -36,7 +36,7 @@ type TiKVInstance struct { } // NewTiKVInstance return a TiKVInstance -func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance, tsos []*PDInstance, isCSEMode bool, cseOptions CSEOptions, isPDMSMode bool) *TiKVInstance { +func NewTiKVInstance(binPath string, dir, host, configPath string, portOffset int, id int, port int, pds []*PDInstance, tsos []*PDInstance, isCSEMode bool, cseOptions CSEOptions, isPDMSMode bool) *TiKVInstance { if port <= 0 { port = 20160 } @@ -46,8 +46,8 @@ func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, port), - StatusPort: utils.MustGetFreePort(host, 20180), + Port: utils.MustGetFreePort(host, port, portOffset), + StatusPort: utils.MustGetFreePort(host, 20180, portOffset), ConfigPath: configPath, }, pds: pds, diff --git a/components/playground/instance/tikv_cdc.go b/components/playground/instance/tikv_cdc.go index cbacb23b2e..8fd717dc32 100644 --- a/components/playground/instance/tikv_cdc.go +++ b/components/playground/instance/tikv_cdc.go @@ -32,14 +32,14 @@ type TiKVCDC struct { var _ Instance = &TiKVCDC{} // NewTiKVCDC create a TiKVCDC instance. -func NewTiKVCDC(binPath string, dir, host, configPath string, id int, pds []*PDInstance) *TiKVCDC { +func NewTiKVCDC(binPath string, dir, host, configPath string, portOffset int, id int, pds []*PDInstance) *TiKVCDC { tikvCdc := &TiKVCDC{ instance: instance{ BinPath: binPath, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, 8600), + Port: utils.MustGetFreePort(host, 8600, portOffset), ConfigPath: configPath, }, pds: pds, diff --git a/components/playground/instance/tiproxy.go b/components/playground/instance/tiproxy.go index c0ed180c90..db43d9284c 100644 --- a/components/playground/instance/tiproxy.go +++ b/components/playground/instance/tiproxy.go @@ -68,7 +68,7 @@ func GenTiProxySessionCerts(dir string) error { } // NewTiProxy create a TiProxy instance. -func NewTiProxy(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiProxy { +func NewTiProxy(binPath string, dir, host, configPath string, portOffset int, id int, port int, pds []*PDInstance) *TiProxy { if port <= 0 { port = 6000 } @@ -78,8 +78,8 @@ func NewTiProxy(binPath string, dir, host, configPath string, id int, port int, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, port), - StatusPort: utils.MustGetFreePort(host, 3080), + Port: utils.MustGetFreePort(host, port, portOffset), + StatusPort: utils.MustGetFreePort(host, 3080, portOffset), ConfigPath: configPath, }, pds: pds, diff --git a/components/playground/main.go b/components/playground/main.go index 47531897a6..2a984afd71 100644 --- a/components/playground/main.go +++ b/components/playground/main.go @@ -75,6 +75,9 @@ type BootOptions struct { Monitor bool `yaml:"monitor"` CSEOpts instance.CSEOptions `yaml:"cse"` // Only available when mode == tidb-cse GrafanaPort int `yaml:"grafana_port"` + PortOffset int `yaml:"port_offset"` + DMMaster instance.Config `yaml:"dm_master"` + DMWorker instance.Config `yaml:"dm_worker"` } var ( @@ -175,11 +178,8 @@ Examples: return err } - port, err := utils.GetFreePort("0.0.0.0", 9527) - if err != nil { - return err - } - err = dumpPort(filepath.Join(dataDir, "port"), port) + port := utils.MustGetFreePort("0.0.0.0", 9527, options.PortOffset) + err := dumpPort(filepath.Join(dataDir, "port"), port) p := NewPlayground(dataDir, port) if err != nil { return err @@ -207,7 +207,7 @@ Examples: sig := (<-sc).(syscall.Signal) atomic.StoreInt32(&p.curSig, int32(sig)) - fmt.Println("Playground receive signal: ", sig) + colorstr.Printf("\n[red][bold]Playground receive signal: %s[reset]\n", sig) // if bootCluster is not done we just cancel context to make it // clean up and return ASAP and exit directly after timeout. @@ -283,6 +283,7 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().BoolVar(&options.Monitor, "monitor", true, "Start prometheus and grafana component") _ = rootCmd.Flags().MarkDeprecated("monitor", "Please use --without-monitor to control whether to disable monitor.") rootCmd.Flags().IntVar(&options.GrafanaPort, "grafana.port", 3000, "grafana port. If not provided, grafana will use 3000 as its port.") + rootCmd.Flags().IntVar(&options.PortOffset, "port-offset", 0, "If specified, all components will use default_port+port_offset as the port. This argument is useful when you want to start multiple playgrounds on the same host. Recommend to set to 10000, 20000, etc.") // NOTE: Do not set default values if they may be changed in different modes. @@ -299,6 +300,8 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().IntVar(&options.TiKVCDC.Num, "kvcdc", 0, "TiKV-CDC instance number") rootCmd.Flags().IntVar(&options.Pump.Num, "pump", 0, "Pump instance number") rootCmd.Flags().IntVar(&options.Drainer.Num, "drainer", 0, "Drainer instance number") + rootCmd.Flags().IntVar(&options.DMMaster.Num, "dm-master", 0, "DM-master instance number") + rootCmd.Flags().IntVar(&options.DMWorker.Num, "dm-worker", 0, "DM-worker instance number") rootCmd.Flags().IntVar(&options.TiDB.UpTimeout, "db.timeout", 60, "TiDB max wait time in seconds for starting, 0 means no limit") rootCmd.Flags().IntVar(&options.TiFlash.UpTimeout, "tiflash.timeout", 120, "TiFlash max wait time in seconds for starting, 0 means no limit") @@ -315,6 +318,10 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().IntVar(&options.TiCDC.Port, "ticdc.port", 0, "Playground TiCDC port. If not provided, TiCDC will use 8300 as its port") rootCmd.Flags().StringVar(&options.TiProxy.Host, "tiproxy.host", "", "Playground TiProxy host. If not provided, TiProxy will still use `host` flag as its host") rootCmd.Flags().IntVar(&options.TiProxy.Port, "tiproxy.port", 0, "Playground TiProxy port. If not provided, TiProxy will use 6000 as its port") + rootCmd.Flags().StringVar(&options.DMMaster.Host, "dm-master.host", "", "DM-master instance host") + rootCmd.Flags().IntVar(&options.DMMaster.Port, "dm-master.port", 8261, "DM-master instance port") + rootCmd.Flags().StringVar(&options.DMWorker.Host, "dm-worker.host", "", "DM-worker instance host") + rootCmd.Flags().IntVar(&options.DMWorker.Port, "dm-worker.port", 8262, "DM-worker instance port") rootCmd.Flags().StringVar(&options.TiDB.ConfigPath, "db.config", "", "TiDB instance configuration file") rootCmd.Flags().StringVar(&options.TiKV.ConfigPath, "kv.config", "", "TiKV instance configuration file") @@ -329,6 +336,8 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.Drainer.ConfigPath, "drainer.config", "", "Drainer instance configuration file") rootCmd.Flags().StringVar(&options.TiCDC.ConfigPath, "ticdc.config", "", "TiCDC instance configuration file") rootCmd.Flags().StringVar(&options.TiKVCDC.ConfigPath, "kvcdc.config", "", "TiKV-CDC instance configuration file") + rootCmd.Flags().StringVar(&options.DMMaster.ConfigPath, "dm-master.config", "", "DM-master instance configuration file") + rootCmd.Flags().StringVar(&options.DMWorker.ConfigPath, "dm-worker.config", "", "DM-worker instance configuration file") rootCmd.Flags().StringVar(&options.TiDB.BinPath, "db.binpath", "", "TiDB instance binary path") rootCmd.Flags().StringVar(&options.TiKV.BinPath, "kv.binpath", "", "TiKV instance binary path") @@ -344,13 +353,15 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.TiKVCDC.BinPath, "kvcdc.binpath", "", "TiKV-CDC instance binary path") rootCmd.Flags().StringVar(&options.Pump.BinPath, "pump.binpath", "", "Pump instance binary path") rootCmd.Flags().StringVar(&options.Drainer.BinPath, "drainer.binpath", "", "Drainer instance binary path") + rootCmd.Flags().StringVar(&options.DMMaster.BinPath, "dm-master.binpath", "", "DM-master instance binary path") + rootCmd.Flags().StringVar(&options.DMWorker.BinPath, "dm-worker.binpath", "", "DM-worker instance binary path") rootCmd.Flags().StringVar(&options.TiKVCDC.Version, "kvcdc.version", "", "TiKV-CDC instance version") - rootCmd.Flags().StringVar(&options.CSEOpts.S3Endpoint, "cse.s3_endpoint", "http://127.0.0.1:9000", "Object store URL for the disaggregated TiFlash, available when --mode=tidb-cse") - rootCmd.Flags().StringVar(&options.CSEOpts.Bucket, "cse.bucket", "tiflash", "Object store bucket for the disaggregated TiFlash, available when --mode=tidb-cse") - rootCmd.Flags().StringVar(&options.CSEOpts.AccessKey, "cse.access_key", "minioadmin", "Object store access key, available when --mode=tidb-cse") - rootCmd.Flags().StringVar(&options.CSEOpts.SecretKey, "cse.secret_key", "minioadmin", "Object store secret key, available when --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.S3Endpoint, "cse.s3_endpoint", "http://127.0.0.1:9000", "Object store URL for --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.Bucket, "cse.bucket", "tiflash", "Object store bucket for --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.AccessKey, "cse.access_key", "minioadmin", "Object store access key for --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.SecretKey, "cse.secret_key", "minioadmin", "Object store secret key for --mode=tidb-cse") rootCmd.AddCommand(newDisplay()) rootCmd.AddCommand(newScaleOut()) @@ -467,6 +478,24 @@ func checkStoreStatus(pdClient *api.PDClient, storeAddr string, timeout int) boo } } +func checkDMMasterStatus(dmMasterClient *api.DMMasterClient, dmMasterAddr string, timeout int) bool { + if timeout > 0 { + for i := 0; i < timeout; i++ { + if _, isActive, _, err := dmMasterClient.GetMaster(dmMasterAddr); err == nil && isActive { + return true + } + time.Sleep(time.Second) + } + return false + } + for { + if _, isActive, _, err := dmMasterClient.GetMaster(dmMasterAddr); err == nil && isActive { + return true + } + time.Sleep(time.Second) + } +} + func hasDashboard(pdAddr string) bool { resp, err := http.Get(fmt.Sprintf("http://%s/dashboard", pdAddr)) if err != nil { @@ -524,11 +553,14 @@ func loadPort(dir string) (port int, err error) { return } -func dumpDSN(fname string, dbs []*instance.TiDBInstance) { +func dumpDSN(fname string, dbs []*instance.TiDBInstance, tdbs []*instance.TiProxy) { var dsn []string for _, db := range dbs { dsn = append(dsn, fmt.Sprintf("mysql://root@%s", db.Addr())) } + for _, tdb := range tdbs { + dsn = append(dsn, fmt.Sprintf("mysql://root@%s", tdb.Addr())) + } _ = utils.WriteFile(fname, []byte(strings.Join(dsn, "\n")), 0644) } diff --git a/components/playground/monitor.go b/components/playground/monitor.go index 3d17a7edae..94c52a09af 100644 --- a/components/playground/monitor.go +++ b/components/playground/monitor.go @@ -78,15 +78,12 @@ func (m *monitor) wait() error { } // the cmd is not started after return -func newMonitor(ctx context.Context, version string, host, dir string) (*monitor, error) { +func newMonitor(ctx context.Context, version string, host, dir string, portOffset int) (*monitor, error) { if err := utils.MkdirAll(dir, 0755); err != nil { return nil, errors.AddStack(err) } - port, err := utils.GetFreePort(host, 9090) - if err != nil { - return nil, err - } + port := utils.MustGetFreePort(host, 9090, portOffset) addr := utils.JoinHostPort(host, port) tmpl := ` @@ -132,6 +129,7 @@ scrape_configs: } var binPath string + var err error if binPath, err = tiupexec.PrepareBinary("prometheus", utils.Version(version), binPath); err != nil { return nil, err } diff --git a/components/playground/ngmonitoring.go b/components/playground/ngmonitoring.go index beb7a58552..851435b53a 100644 --- a/components/playground/ngmonitoring.go +++ b/components/playground/ngmonitoring.go @@ -45,16 +45,12 @@ func (m *ngMonitoring) wait() error { } // the cmd is not started after return -func newNGMonitoring(ctx context.Context, version string, host, dir string, pds []*instance.PDInstance) (*ngMonitoring, error) { +func newNGMonitoring(ctx context.Context, version string, host, dir string, portOffset int, pds []*instance.PDInstance) (*ngMonitoring, error) { if err := utils.MkdirAll(dir, 0755); err != nil { return nil, errors.AddStack(err) } - port, err := utils.GetFreePort(host, 12020) - if err != nil { - return nil, err - } - + port := utils.MustGetFreePort(host, 12020, portOffset) m := new(ngMonitoring) var endpoints []string for _, pd := range pds { diff --git a/components/playground/playground.go b/components/playground/playground.go index 446cd61992..569b3b55a4 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -41,6 +41,7 @@ import ( "github.com/pingcap/tiup/pkg/environment" logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/tidbver" + "github.com/pingcap/tiup/pkg/tui/colorstr" "github.com/pingcap/tiup/pkg/tui/progress" "github.com/pingcap/tiup/pkg/utils" "golang.org/x/mod/semver" @@ -70,6 +71,8 @@ type Playground struct { tikvCdcs []*instance.TiKVCDC pumps []*instance.Pump drainers []*instance.Drainer + dmMasters []*instance.DMMaster + dmWorkers []*instance.DMWorker startedInstances []instance.Instance idAlloc map[string]int @@ -136,6 +139,15 @@ func (p *Playground) binlogClient() (*api.BinlogClient, error) { return api.NewBinlogClient(addrs, 5*time.Second, nil) } +func (p *Playground) dmMasterClient() *api.DMMasterClient { + var addrs []string + for _, inst := range p.dmMasters { + addrs = append(addrs, inst.Addr()) + } + + return api.NewDMMasterClient(addrs, 5*time.Second, nil) +} + func (p *Playground) pdClient() *api.PDClient { var addrs []string for _, inst := range p.pds { @@ -381,6 +393,14 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { return nil } } + case spec.ComponentDMWorker: + if err := p.handleScaleInDMWorker(pid); err != nil { + return err + } + case spec.ComponentDMMaster: + if err := p.handleScaleInDMMaster(pid); err != nil { + return err + } default: fmt.Fprintf(w, "unknown component in scale in: %s", cid) return nil @@ -398,6 +418,38 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { return nil } +func (p *Playground) handleScaleInDMWorker(pid int) error { + for i := 0; i < len(p.dmWorkers); i++ { + if p.dmWorkers[i].Pid() == pid { + inst := p.dmWorkers[i] + + c := p.dmMasterClient() + if err := c.OfflineWorker(inst.Name(), nil); err != nil { + return err + } + p.dmWorkers = append(p.dmWorkers[:i], p.dmWorkers[i+1:]...) + return nil + } + } + return nil +} + +func (p *Playground) handleScaleInDMMaster(pid int) error { + for i := 0; i < len(p.dmMasters); i++ { + if p.dmMasters[i].Pid() == pid { + inst := p.dmMasters[i] + + c := p.dmMasterClient() + if err := c.OfflineMaster(inst.Name(), nil); err != nil { + return err + } + p.dmMasters = append(p.dmMasters[:i], p.dmMasters[i+1:]...) + return nil + } + } + return nil +} + func (p *Playground) sanitizeConfig(boot instance.Config, cfg *instance.Config) error { if cfg.BinPath == "" { cfg.BinPath = boot.BinPath @@ -441,6 +493,10 @@ func (p *Playground) sanitizeComponentConfig(cid string, cfg *instance.Config) e return p.sanitizeConfig(p.bootOptions.Drainer, cfg) case spec.ComponentTiProxy: return p.sanitizeConfig(p.bootOptions.TiProxy, cfg) + case spec.ComponentDMMaster: + return p.sanitizeConfig(p.bootOptions.DMMaster, cfg) + case spec.ComponentDMWorker: + return p.sanitizeConfig(p.bootOptions.DMWorker, cfg) default: return fmt.Errorf("unknown %s in sanitizeConfig", cid) } @@ -458,7 +514,7 @@ func (p *Playground) startInstance(ctx context.Context, inst instance.Instance) return err } - if err := inst.PrepareBinary(component, version); err != nil { + if err := inst.PrepareBinary(component, inst.Component(), version); err != nil { return err } @@ -681,6 +737,20 @@ func (p *Playground) WalkInstances(fn func(componentID string, ins instance.Inst } } + for _, ins := range p.dmMasters { + err := fn(spec.ComponentDMMaster, ins) + if err != nil { + return err + } + } + + for _, ins := range p.dmWorkers { + err := fn(spec.ComponentDMWorker, ins) + if err != nil { + return err + } + } + return nil } @@ -722,7 +792,7 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif switch componentID { case spec.ComponentPD: - inst := instance.NewPDInstance(pdRole, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") + inst := instance.NewPDInstance(pdRole, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") ins = inst if pdRole == instance.PDRoleNormal || pdRole == instance.PDRoleAPI { if p.booted { @@ -740,48 +810,59 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif p.schedulings = append(p.schedulings, inst) } case spec.ComponentTSO: - inst := instance.NewPDInstance(instance.PDRoleTSO, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") + inst := instance.NewPDInstance(instance.PDRoleTSO, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") ins = inst p.tsos = append(p.tsos, inst) case spec.ComponentScheduling: - inst := instance.NewPDInstance(instance.PDRoleScheduling, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") + inst := instance.NewPDInstance(instance.PDRoleScheduling, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") ins = inst p.schedulings = append(p.schedulings, inst) case spec.ComponentTiDB: - inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, dataDir, p.enableBinlog(), p.bootOptions.Mode == "tidb-cse") + inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds, dataDir, p.enableBinlog(), p.bootOptions.Mode == "tidb-cse") ins = inst p.tidbs = append(p.tidbs, inst) case spec.ComponentTiKV: - inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, p.tsos, p.bootOptions.Mode == "tidb-cse", p.bootOptions.CSEOpts, p.bootOptions.PDMode == "ms") + inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds, p.tsos, p.bootOptions.Mode == "tidb-cse", p.bootOptions.CSEOpts, p.bootOptions.PDMode == "ms") ins = inst p.tikvs = append(p.tikvs, inst) case spec.ComponentTiFlash: - inst := instance.NewTiFlashInstance(tiflashRole, p.bootOptions.CSEOpts, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, p.tidbs, cfg.Version) + inst := instance.NewTiFlashInstance(tiflashRole, p.bootOptions.CSEOpts, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, p.tidbs, cfg.Version) ins = inst p.tiflashs = append(p.tiflashs, inst) case spec.ComponentTiProxy: if err := instance.GenTiProxySessionCerts(dataDir); err != nil { return nil, err } - inst := instance.NewTiProxy(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds) + inst := instance.NewTiProxy(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds) ins = inst p.tiproxys = append(p.tiproxys, inst) case spec.ComponentCDC: - inst := instance.NewTiCDC(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds) + inst := instance.NewTiCDC(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds) ins = inst p.ticdcs = append(p.ticdcs, inst) case spec.ComponentTiKVCDC: - inst := instance.NewTiKVCDC(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) + inst := instance.NewTiKVCDC(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds) ins = inst p.tikvCdcs = append(p.tikvCdcs, inst) case spec.ComponentPump: - inst := instance.NewPump(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) + inst := instance.NewPump(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds) ins = inst p.pumps = append(p.pumps, inst) case spec.ComponentDrainer: - inst := instance.NewDrainer(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) + inst := instance.NewDrainer(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds) ins = inst p.drainers = append(p.drainers, inst) + case spec.ComponentDMMaster: + inst := instance.NewDMMaster(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port) + ins = inst + p.dmMasters = append(p.dmMasters, inst) + for _, master := range p.dmMasters { + master.SetInitEndpoints(p.dmMasters) + } + case spec.ComponentDMWorker: + inst := instance.NewDMWorker(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.dmMasters) + ins = inst + p.dmWorkers = append(p.dmWorkers, inst) default: return nil, errors.Errorf("unknown component: %s", componentID) } @@ -797,13 +878,13 @@ func (p *Playground) waitAllDBUp() ([]string, []string) { var tidbMu, tiproxyMu sync.Mutex var bars *progress.MultiBar if len(p.tiproxys) > 0 { - bars = progress.NewMultiBar(color.YellowString("Waiting for tidb and tiproxy instances ready")) + bars = progress.NewMultiBar(colorstr.Sprintf("[dark_gray]Waiting for tidb and tiproxy instances ready")) } else { - bars = progress.NewMultiBar(color.YellowString("Waiting for tidb instances ready")) + bars = progress.NewMultiBar(colorstr.Sprintf("[dark_gray]Waiting for tidb instances ready")) } for _, db := range p.tidbs { wg.Add(1) - prefix := color.YellowString(db.Addr()) + prefix := "- TiDB: " + db.Addr() bar := bars.AddBar(prefix) go func(dbInst *instance.TiDBInstance) { defer wg.Done() @@ -827,7 +908,7 @@ func (p *Playground) waitAllDBUp() ([]string, []string) { } for _, db := range p.tiproxys { wg.Add(1) - prefix := color.YellowString(db.Addr()) + prefix := "- TiProxy: " + db.Addr() bar := bars.AddBar(prefix) go func(dbInst *instance.TiProxy) { defer wg.Done() @@ -868,10 +949,18 @@ func (p *Playground) waitAllTiFlashUp() { ) var wg sync.WaitGroup - bars := progress.NewMultiBar(color.YellowString("Waiting for tiflash instances ready")) + bars := progress.NewMultiBar(colorstr.Sprintf("[dark_gray]Waiting for tiflash instances ready")) for _, flash := range p.tiflashs { wg.Add(1) - prefix := color.YellowString(flash.Addr()) + + tiflashKindName := "TiFlash" + if flash.Role == instance.TiFlashRoleDisaggCompute { + tiflashKindName = "TiFlash (CN)" + } else if flash.Role == instance.TiFlashRoleDisaggWrite { + tiflashKindName = "TiFlash (WN)" + } + + prefix := fmt.Sprintf("- %s: %s", tiflashKindName, flash.Addr()) bar := bars.AddBar(prefix) go func(flashInst *instance.TiFlashInstance) { defer wg.Done() @@ -899,6 +988,40 @@ func (p *Playground) waitAllTiFlashUp() { } } +func (p *Playground) waitAllDMMasterUp() { + if len(p.dmMasters) > 0 { + var wg sync.WaitGroup + bars := progress.NewMultiBar(colorstr.Sprintf("[dark_gray]Waiting for dm-master instances ready")) + for _, master := range p.dmMasters { + wg.Add(1) + prefix := master.Addr() + bar := bars.AddBar(prefix) + go func(masterInst *instance.DMMaster) { + defer wg.Done() + displayResult := &progress.DisplayProps{ + Prefix: prefix, + } + if cmd := masterInst.Cmd(); cmd == nil { + displayResult.Mode = progress.ModeError + displayResult.Suffix = "initialize command failed" + } else if state := cmd.ProcessState; state != nil && state.Exited() { + displayResult.Mode = progress.ModeError + displayResult.Suffix = fmt.Sprintf("process exited with code: %d", state.ExitCode()) + } else if s := checkDMMasterStatus(p.dmMasterClient(), masterInst.Name(), options.DMMaster.UpTimeout); !s { + displayResult.Mode = progress.ModeError + displayResult.Suffix = "failed to up after timeout" + } else { + displayResult.Mode = progress.ModeDone + } + bar.UpdateDisplay(displayResult) + }(master) + } + bars.StartRenderLoop() + wg.Wait() + bars.StopRenderLoop() + } +} + func (p *Playground) bindVersion(comp string, version string) (bindVersion string) { bindVersion = version switch comp { @@ -927,6 +1050,8 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme &options.Pump, &options.Drainer, &options.TiKVCDC, + &options.DMMaster, + &options.DMWorker, } { path, err := getAbsolutePath(cfg.ConfigPath) if err != nil { @@ -937,8 +1062,8 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme p.bootOptions = options - // All others components depend on the pd, we just ensure the pd count must be great than 0 - if options.PDMode != "ms" && options.PD.Num < 1 { + // All others components depend on the pd except dm, we just ensure the pd count must be great than 0 + if options.PDMode != "ms" && options.PD.Num < 1 && options.DMMaster.Num < 1 { return fmt.Errorf("all components count must be great than 0 (pd=%v)", options.PD.Num) } @@ -968,6 +1093,8 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme {spec.ComponentCDC, "", "", options.TiCDC}, {spec.ComponentTiKVCDC, "", "", options.TiKVCDC}, {spec.ComponentDrainer, "", "", options.Drainer}, + {spec.ComponentDMMaster, "", "", options.DMMaster}, + {spec.ComponentDMWorker, "", "", options.DMWorker}, } if options.Mode == "tidb" { @@ -1014,7 +1141,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme // Try to create bucket. err := s3Client.MakeBucket(ctxCheck, options.CSEOpts.Bucket, minio.MakeBucketOptions{}) if err != nil { - return fmt.Errorf("CSE mode preflight check failed: Bucket %s doesn't exist", options.CSEOpts.Bucket) + return fmt.Errorf("CSE mode preflight check failed: Bucket %s doesn't exist and fail to create automatically (your bucket name may be invalid?)", options.CSEOpts.Bucket) } } @@ -1051,11 +1178,17 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme } anyPumpReady := false + allDMMasterReady := false // Start all instance except tiflash. err := p.WalkInstances(func(cid string, ins instance.Instance) error { if cid == spec.ComponentTiFlash { return nil } + // wait dm-master up before dm-worker + if cid == spec.ComponentDMWorker && !allDMMasterReady { + p.waitAllDMMasterUp() + allDMMasterReady = true + } err := p.startInstance(ctx, ins) if err != nil { @@ -1134,9 +1267,20 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme } } - if pdAddr := p.pds[0].Addr(); len(p.tidbs) > 0 && hasDashboard(pdAddr) { - fmt.Printf("TiDB Dashboard: ") - colorCmd.Printf("http://%s/dashboard\n", pdAddr) + if len(p.dmMasters) > 0 { + fmt.Printf("Connect DM: ") + endpoints := make([]string, 0, len(p.dmMasters)) + for _, dmMaster := range p.dmMasters { + endpoints = append(endpoints, dmMaster.Addr()) + } + colorCmd.Printf("tiup dmctl --master-addr %s\n", strings.Join(endpoints, ",")) + } + + if len(p.pds) > 0 { + if pdAddr := p.pds[0].Addr(); len(p.tidbs) > 0 && hasDashboard(pdAddr) { + fmt.Printf("TiDB Dashboard: ") + colorCmd.Printf("http://%s/dashboard\n", pdAddr) + } } if p.bootOptions.Mode == "tikv-slim" { @@ -1176,7 +1320,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme p.updateMonitorTopology(spec.ComponentPrometheus, *monitorInfo) } - dumpDSN(filepath.Join(p.dataDir, "dsn"), p.tidbs) + dumpDSN(filepath.Join(p.dataDir, "dsn"), p.tidbs, p.tiproxys) go func() { // fmt.Printf("serve at :%d\n", p.port) @@ -1228,9 +1372,9 @@ func (p *Playground) wait() error { func (p *Playground) terminate(sig syscall.Signal) { kill := func(name string, pid int, wait func() error) { if sig == syscall.SIGKILL { - fmt.Printf("Force %s(%d) to quit...\n", name, pid) + colorstr.Printf("[dark_gray]Force %s(%d) to quit...\n", name, pid) } else if atomic.LoadInt32(&p.curSig) == int32(sig) { // In case of double ctr+c - fmt.Printf("Wait %s(%d) to quit...\n", name, pid) + colorstr.Printf("[dark_gray]Wait %s(%d) to quit...\n", name, pid) } _ = syscall.Kill(pid, sig) @@ -1253,6 +1397,19 @@ func (p *Playground) terminate(sig syscall.Signal) { if p.grafana != nil && p.grafana.cmd != nil && p.grafana.cmd.Process != nil { go kill("grafana", p.grafana.cmd.Process.Pid, p.grafana.wait) } + + for _, inst := range p.dmWorkers { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { + kill(inst.Component(), inst.Pid(), inst.Wait) + } + } + + for _, inst := range p.dmMasters { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { + kill(inst.Component(), inst.Pid(), inst.Wait) + } + } + for _, inst := range p.tiflashs { if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) @@ -1346,7 +1503,7 @@ func (p *Playground) bootMonitor(ctx context.Context, env *environment.Environme dataDir := p.dataDir promDir := filepath.Join(dataDir, "prometheus") - monitor, err := newMonitor(ctx, options.Version, options.Host, promDir) + monitor, err := newMonitor(ctx, options.Version, options.Host, promDir, options.PortOffset) if err != nil { return nil, nil, err } @@ -1390,7 +1547,7 @@ func (p *Playground) bootNGMonitoring(ctx context.Context, env *environment.Envi dataDir := p.dataDir promDir := filepath.Join(dataDir, "prometheus") - ngm, err := newNGMonitoring(ctx, options.Version, options.Host, promDir, p.pds) + ngm, err := newNGMonitoring(ctx, options.Version, options.Host, promDir, options.PortOffset, p.pds) if err != nil { return nil, err } @@ -1469,7 +1626,7 @@ func (p *Playground) bootGrafana(ctx context.Context, env *environment.Environme grafana := newGrafana(options.Version, options.Host, options.GrafanaPort) // fmt.Println("Start Grafana instance...") - err = grafana.start(ctx, grafanaDir, "http://"+utils.JoinHostPort(monitorInfo.IP, monitorInfo.Port)) + err = grafana.start(ctx, grafanaDir, options.PortOffset, "http://"+utils.JoinHostPort(monitorInfo.IP, monitorInfo.Port)) if err != nil { return nil, err } diff --git a/docker/control/Dockerfile b/docker/control/Dockerfile index fe2901c994..b70c7939cd 100644 --- a/docker/control/Dockerfile +++ b/docker/control/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17-bullseye +FROM golang:1.21 # Use mirrors for poor network... diff --git a/docker/up.sh b/docker/up.sh index 468a05a90a..dc7097a4b4 100755 --- a/docker/up.sh +++ b/docker/up.sh @@ -182,9 +182,6 @@ exists python || exists docker || { ERROR "Please install docker (https://docs.docker.com/engine/installation/)"; exit 1; } -exists docker-compose || - { ERROR "Please install docker-compose (https://docs.docker.com/compose/install/)"; - exit 1; } exists pip || { @@ -233,17 +230,17 @@ echo "TIUP_TEST_IP_PREFIX=$ipprefix" >> ./secret/control.env INFO "Running \`docker-compose build\`" # shellcheck disable=SC2086 -docker-compose -f docker-compose.yml ${COMPOSE} ${DEV} build +docker compose -f docker-compose.yml ${COMPOSE} ${DEV} build INFO "Running \`docker-compose up\`" if [ "${RUN_AS_DAEMON}" -eq 1 ]; then # shellcheck disable=SC2086 - docker-compose -f docker-compose.yml ${COMPOSE} ${DEV} up -d + docker compose -f docker-compose.yml ${COMPOSE} ${DEV} up -d INFO "All containers started, run \`docker ps\` to view" else INFO "Please run \`docker exec -it tiup-cluster-control bash\` in another terminal to proceed" # shellcheck disable=SC2086 - docker-compose -f docker-compose.yml ${COMPOSE} ${DEV} up + docker compose -f docker-compose.yml ${COMPOSE} ${DEV} up fi popd diff --git a/embed/examples/cluster/topology.example.yaml b/embed/examples/cluster/topology.example.yaml index ce9953c783..a3ec4a9299 100644 --- a/embed/examples/cluster/topology.example.yaml +++ b/embed/examples/cluster/topology.example.yaml @@ -300,6 +300,32 @@ kvcdc_servers: data_dir: "/data1/tidb-data/tikv-cdc-8600" log_dir: "/data1/tidb-deploy/tikv-cdc-8600/log" +# # Server configs are used to specify the configuration of TiCDC Servers. +#cdc_servers: +# - host: 10.0.1.20 + # # SSH port of the server. + # ssh_port: 22 + # # TiCDC Server communication port. + # port: 8300 + # # TiCDC Server data storage directory. + # data_dir: "/data1/tidb-deploy/cdc-8300" + # # TiCDC Server log file storage directory. + # log_dir: "/data1/tidb-deploy/cdc-8300/log" + # # TiCDC Server deployment file, startup script, configuration file storage directory. + # deploy_dir: "/data1/tidb-deploy/cdc-8300" + # gc-ttl: 86400 # 24h + # tz: "System" + # numa_node: "0,1" + # config: + # log.level: warn + # ticdc_cluster_id: "default" + # - host: 10.0.1.21 + # ssh_port: 22 + # port: 8301 + # data_dir: "/data2/tidb-deploy/cdc-8301" + # log_dir: "/data2/tidb-deploy/cdc-8301/log" + # deploy_dir: "/data2/tidb-deploy/cdc-8301" + # # Server configs are used to specify the configuration of TiDB Dashboard Servers. Available from v6.5.0 # tidb_dashboard_servers: # # The ip address of the PD Server. @@ -315,9 +341,8 @@ kvcdc_servers: # # PD Server log file storage directory. # log_dir: "/tidb-deploy/tidb-dashboard-12333/log" # # numa node bindings. - # numa_node: "0,1" - - + # numa_node: "0,1" + # # Server configs are used to specify the configuration of Prometheus Server. monitoring_servers: # # The ip address of the Monitoring Server. diff --git a/embed/templates/config/grafana.ini.tpl b/embed/templates/config/grafana.ini.tpl index 9cc5140901..602cbce620 100644 --- a/embed/templates/config/grafana.ini.tpl +++ b/embed/templates/config/grafana.ini.tpl @@ -129,7 +129,7 @@ check_for_updates = true admin_user = {{.Username}} # default admin password, can be changed before first start of grafana, or in profile settings -admin_password = {{.Password}} +admin_password = `{{.Password}}` # used for signing ;secret_key = SW2YcwTIb9zpOOhoPsMm diff --git a/embed/templates/scripts/run_node_exporter.sh.tpl b/embed/templates/scripts/run_node_exporter.sh.tpl index 9634eed2bc..02acb1491e 100644 --- a/embed/templates/scripts/run_node_exporter.sh.tpl +++ b/embed/templates/scripts/run_node_exporter.sh.tpl @@ -21,10 +21,10 @@ exec $EXPORTER_BIN \ {{- end}} --web.listen-address=":{{.Port}}" \ --collector.tcpstat \ - --collector.systemd \ --collector.mountstats \ --collector.meminfo_numa \ - --collector.interrupts \ --collector.buddyinfo \ --collector.vmstat.fields="^.*" \ --log.level="info" + #--collector.systemd \ + #--collector.interrupts \ diff --git a/embed/templates/scripts/run_scheduling.sh.tpl b/embed/templates/scripts/run_scheduling.sh.tpl index a15b1ba4f7..2ba72fed0f 100644 --- a/embed/templates/scripts/run_scheduling.sh.tpl +++ b/embed/templates/scripts/run_scheduling.sh.tpl @@ -11,6 +11,9 @@ cd "${DEPLOY_DIR}" || exit 1 exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server services scheduling\ {{- else}} exec env GODEBUG=madvdontneed=1 bin/pd-server services scheduling \ +{{- end}} +{{- if .Name}} + --name="{{.Name}}" \ {{- end}} --backend-endpoints="{{.BackendEndpoints}}" \ --listen-addr="{{.ListenURL}}" \ diff --git a/embed/templates/scripts/run_tso.sh.tpl b/embed/templates/scripts/run_tso.sh.tpl index 0d6486d73e..177b676aff 100644 --- a/embed/templates/scripts/run_tso.sh.tpl +++ b/embed/templates/scripts/run_tso.sh.tpl @@ -11,6 +11,9 @@ cd "${DEPLOY_DIR}" || exit 1 exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server services tso\ {{- else}} exec env GODEBUG=madvdontneed=1 bin/pd-server services tso \ +{{- end}} +{{- if .Name}} + --name="{{.Name}}" \ {{- end}} --backend-endpoints="{{.BackendEndpoints}}" \ --listen-addr="{{.ListenURL}}" \ diff --git a/install.sh b/install.sh index bd40111c2e..ed1bc27e22 100755 --- a/install.sh +++ b/install.sh @@ -64,11 +64,15 @@ fi chmod 755 "$bin_dir/tiup" -"$bin_dir/tiup" mirror set $repo +"$bin_dir/tiup" mirror set $repo --silent bold=$(tput bold 2>/dev/null) +green=$(tput setaf 2 2>/dev/null) +cyan=$(tput setaf 6 2>/dev/null) sgr0=$(tput sgr0 2>/dev/null) +echo + # Refrence: https://stackoverflow.com/questions/14637979/how-to-permanently-set-path-on-linux-unix shell=$(echo $SHELL | awk 'BEGIN {FS="/";} { print $NF }') echo "Detected shell: ${bold}$shell${sgr0}" @@ -82,16 +86,21 @@ else PROFILE=${HOME}/.profile fi echo "Shell profile: ${bold}$PROFILE${sgr0}" - +echo +echo "${bold}${green}✔ ${sgr0}Installed in ${bold}$bin_dir/tiup${sgr0}" case :$PATH: in - *:$bin_dir:*) echo "PATH already contains $bin_dir" ;; + *:$bin_dir:*) echo "${bold}${green}✔ ${sgr0}tiup PATH is already set, skip" ;; *) printf '\nexport PATH=%s:$PATH\n' "$bin_dir" >> "$PROFILE" - echo "$PROFILE has been modified to add tiup to PATH" - echo "open a new terminal or ${bold}source ${PROFILE}${sgr0} to use it" + echo "${bold}${green}✔ ${sgr0}Added tiup PATH into ${bold}${shell}${sgr0} profile" ;; esac - -echo "Installed path: ${bold}$bin_dir/tiup${sgr0}" -echo "===============================================" -echo "Have a try: ${bold}tiup playground${sgr0}" -echo "===============================================" +echo +echo "${bold}tiup is installed now${sgr0} 🎉" +echo +echo Next step: +echo +echo " 1: To make PATH change effective, restart your shell or execute:" +echo " ${bold}${cyan}source ${PROFILE}${sgr0}" +echo +echo " 2: Start a local TiDB for development:" +echo " ${bold}${cyan}tiup playground${sgr0}" diff --git a/pkg/cluster/manager/display.go b/pkg/cluster/manager/display.go index 5fabc8072e..32821342aa 100644 --- a/pkg/cluster/manager/display.go +++ b/pkg/cluster/manager/display.go @@ -20,7 +20,6 @@ import ( "errors" "fmt" "math" - "net/url" "sort" "strconv" "strings" @@ -102,6 +101,7 @@ type ClusterMetaInfo struct { TLSClientCert string `json:"tls_client_cert,omitempty"` TLSClientKey string `json:"tls_client_key,omitempty"` DashboardURL string `json:"dashboard_url,omitempty"` + DashboardURLS []string `json:"dashboard_urls,omitempty"` GrafanaURLS []string `json:"grafana_urls,omitempty"` } @@ -157,6 +157,7 @@ func (m *Manager) Display(dopt DisplayOption, opt operator.Options) error { "", // Client Key "", nil, + nil, }, InstanceInfos: clusterInstInfos, } @@ -261,26 +262,13 @@ func (m *Manager) Display(dopt DisplayOption, opt operator.Options) error { return err } - var dashboardAddr string ctx := ctxt.New( context.Background(), opt.Concurrency, m.logger, ) if t, ok := topo.(*spec.Specification); ok { - var err error - dashboardAddr, err = t.GetDashboardAddress(ctx, tlsCfg, statusTimeout, masterActive...) - if err == nil && !set.NewStringSet("", "auto", "none").Exist(dashboardAddr) { - scheme := "http" - if tlsCfg != nil { - scheme = "https" - } - if m.logger.GetDisplayMode() == logprinter.DisplayModeJSON { - j.ClusterMetaInfo.DashboardURL = fmt.Sprintf("%s://%s/dashboard", scheme, dashboardAddr) - } else { - fmt.Printf("Dashboard URL: %s\n", cyan.Sprintf("%s://%s/dashboard", scheme, dashboardAddr)) - } - } + _ = m.displayDashboards(ctx, t, j, statusTimeout, tlsCfg, "", masterActive...) } if m.logger.GetDisplayMode() == logprinter.DisplayModeJSON { @@ -386,6 +374,7 @@ func (m *Manager) DisplayTiKVLabels(dopt DisplayOption, opt operator.Options) er "", // Client Key "", nil, + nil, }, } @@ -582,7 +571,7 @@ func (m *Manager) GetClusterTopology(dopt DisplayOption, opt operator.Options) ( var dashboardAddr string if t, ok := topo.(*spec.Specification); ok { - dashboardAddr, _ = t.GetDashboardAddress(ctx, tlsCfg, statusTimeout, masterActive...) + dashboardAddr, _ = t.GetPDDashboardAddress(ctx, tlsCfg, statusTimeout, masterActive...) } clusterInstInfos := []InstInfo{} @@ -801,25 +790,28 @@ func (m *Manager) DisplayDashboardInfo(clusterName string, timeout time.Duration } ctx := context.WithValue(context.Background(), logprinter.ContextKeyLogger, m.logger) - pdAPI := api.NewPDClient(ctx, metadata.Topology.GetPDListWithManageHost(), timeout, tlsCfg) - dashboardAddr, err := pdAPI.GetDashboardAddress() - if err != nil { - return fmt.Errorf("failed to retrieve TiDB Dashboard instance from PD: %s", err) - } - if dashboardAddr == "auto" { - return fmt.Errorf("TiDB Dashboard is not initialized, please start PD and try again") - } else if dashboardAddr == "none" { - return fmt.Errorf("TiDB Dashboard is disabled") - } + return m.displayDashboards(ctx, metadata.Topology, nil, timeout, tlsCfg, clusterName, metadata.Topology.GetPDListWithManageHost()...) +} - u, err := url.Parse(dashboardAddr) - if err != nil { - return fmt.Errorf("unknown TiDB Dashboard PD instance: %s", dashboardAddr) +func (m *Manager) displayDashboards(ctx context.Context, t *spec.Specification, j *JSONOutput, timeout time.Duration, tlsCfg *tls.Config, clusterName string, pdList ...string) error { + dashboardAddrs := []string{} + t.IterInstance(func(ins spec.Instance) { + if ins.Role() != spec.ComponentDashboard { + return + } + dashboardAddrs = append(dashboardAddrs, utils.JoinHostPort(ins.GetManageHost(), ins.GetPort())) + }) + + pdDashboardAddr, err := t.GetPDDashboardAddress(ctx, tlsCfg, timeout, pdList...) + if err == nil && !set.NewStringSet("", "auto", "none").Exist(pdDashboardAddr) { + dashboardAddrs = append(dashboardAddrs, pdDashboardAddr) } - u.Path = "/dashboard/" + if len(dashboardAddrs) == 0 { + return fmt.Errorf("TiDB Dashboard is missing, try again later") + } - if tlsCfg != nil { + if clusterName != "" && tlsCfg != nil { fmt.Println( "Client certificate:", color.CyanString(m.specManager.Path(clusterName, spec.TLSCertKeyDir, spec.PFXClientCert)), @@ -829,10 +821,32 @@ func (m *Manager) DisplayDashboardInfo(clusterName string, timeout time.Duration color.CyanString(crypto.PKCS12Password), ) } - fmt.Println( - "Dashboard URL:", - color.CyanString(u.String()), - ) + + for i, addr := range dashboardAddrs { + scheme := "http" + + // show the original info + if addr == pdDashboardAddr { + if tlsCfg != nil { + scheme = "https" + } + if m.logger.GetDisplayMode() == logprinter.DisplayModeJSON && j != nil { + j.ClusterMetaInfo.DashboardURL = fmt.Sprintf("%s://%s/dashboard", scheme, addr) + } else { + fmt.Printf("Dashboard URL: %s\n", color.CyanString("%s://%s/dashboard", scheme, addr)) + } + } + + if m.logger.GetDisplayMode() == logprinter.DisplayModeJSON && j != nil { + j.ClusterMetaInfo.DashboardURLS = append(j.ClusterMetaInfo.DashboardURLS, fmt.Sprintf("%s://%s/dashboard", scheme, addr)) + } else { + dashboardAddrs[i] = color.CyanString("%s://%s/dashboard", scheme, addr) + } + } + + if m.logger.GetDisplayMode() != logprinter.DisplayModeJSON || j == nil { + fmt.Printf("Dashboard URLs: %s\n", strings.Join(dashboardAddrs, ",")) + } return nil } diff --git a/pkg/cluster/manager/manager_test.go b/pkg/cluster/manager/manager_test.go index e80d1d1c64..b82b6eb785 100644 --- a/pkg/cluster/manager/manager_test.go +++ b/pkg/cluster/manager/manager_test.go @@ -85,6 +85,21 @@ pd_servers: assert.Nil(err) err = validateNewTopo(&topo) assert.NotNil(err) + + topo = spec.Specification{} + err = yaml.Unmarshal([]byte(` +global: + user: "test4" + deploy_dir: "test-deploy" + data_dir: "test-data" +tso_servers: + - host: 172.16.5.53 +scheduling_servers: + - host: 172.16.5.54 +`), &topo) + assert.Nil(err) + err = validateNewTopo(&topo) + assert.Nil(err) } func TestDeduplicateCheckResult(t *testing.T) { diff --git a/pkg/cluster/manager/transfer_test.go b/pkg/cluster/manager/transfer_test.go index bef179dd0b..d4b4581f6a 100644 --- a/pkg/cluster/manager/transfer_test.go +++ b/pkg/cluster/manager/transfer_test.go @@ -53,4 +53,32 @@ func TestRenderSpec(t *testing.T) { dir, err = renderSpec("{{.DataDir}}", s, "test-pd") assert.Nil(t, err) assert.NotEmpty(t, dir) + + s = &spec.TSOInstance{BaseInstance: spec.BaseInstance{ + InstanceSpec: &spec.TSOSpec{ + Host: "172.16.5.140", + SSHPort: 22, + Name: "tso-1", + DeployDir: "/home/test/deploy/tso-3379", + DataDir: "/home/test/deploy/tso-3379/data", + }, + }} + // s.BaseInstance.InstanceSpec + dir, err = renderSpec("{{.DataDir}}", s, "test-tso") + assert.Nil(t, err) + assert.NotEmpty(t, dir) + + s = &spec.SchedulingInstance{BaseInstance: spec.BaseInstance{ + InstanceSpec: &spec.SchedulingSpec{ + Host: "172.16.5.140", + SSHPort: 22, + Name: "scheduling-1", + DeployDir: "/home/test/deploy/scheduling-3379", + DataDir: "/home/test/deploy/scheduling-3379/data", + }, + }} + // s.BaseInstance.InstanceSpec + dir, err = renderSpec("{{.DataDir}}", s, "test-scheduling") + assert.Nil(t, err) + assert.NotEmpty(t, dir) } diff --git a/pkg/cluster/operation/check.go b/pkg/cluster/operation/check.go index f849c8b356..f4ca75b764 100644 --- a/pkg/cluster/operation/check.go +++ b/pkg/cluster/operation/check.go @@ -159,6 +159,11 @@ func checkSysInfo(opt *CheckOptions, sysInfo *sysinfo.SysInfo) []*CheckResult { return results } +// Try to keep this in sync with +// https://docs.pingcap.com/tidb/stable/hardware-and-software-requirements#os-and-platform-requirements +// +// This information is in most cases based on the `ID` (Vendor) and `VERSION_ID` (Release) of /etc/os-release +// See https://github.com/AstroProfundis/sysinfo/blob/tiup/os.go for details. func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { result := &CheckResult{ Name: CheckNameOSVer, @@ -175,17 +180,31 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { return result } case "amzn": + // https://aws.amazon.com/linux/amazon-linux-2023/ + if osInfo.Version == "2023" { + return result + } + // Amazon Linux 2 is based on CentOS 7 and is recommended for // AWS Graviton 2 (ARM64) deployments. + // https://aws.amazon.com/amazon-linux-2/ if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 2 || ver >= 3 { - result.Err = fmt.Errorf("%s %s not supported, use version 2 please", + result.Err = fmt.Errorf("%s %s not supported, use Amazon Linux 2 or Amazon Linux 2023 please", osInfo.Name, osInfo.Release) return result } - case "centos", "redhat", "rhel", "ol": - // check version - if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 7 { - result.Err = fmt.Errorf("%s %s not supported, use version 8 please", + case "centos": + // CentOS Linux is EOL + // CentOS Stream 9 and newer is still fine + if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 9 { + result.Err = fmt.Errorf("%s %s not supported, use version 9 or higher", + osInfo.Name, osInfo.Release) + return result + } + case "redhat", "rhel", "ol": + // RHEL 8.4 or newer 8.x versions are supported + if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 8.4 || ver >= 9 { + result.Err = fmt.Errorf("%s %s not supported, use version 8.4 or a later 8.x version please", osInfo.Name, osInfo.Release) return result } @@ -198,22 +217,22 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { } case "debian": // debian support is not fully tested, but we suppose it should work - msg := "debian support is not fully tested, be careful" + msg := "Debian support is not fully tested, be careful" result.Err = fmt.Errorf("%s (%s)", result.Msg, msg) result.Warn = true - if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 9 { - result.Err = fmt.Errorf("%s %s not supported, use version 9 or higher (%s)", + if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 10 { + result.Err = fmt.Errorf("%s %s not supported, use version 10 or higher (%s)", osInfo.Name, osInfo.Release, msg) result.Warn = false return result } case "ubuntu": // ubuntu support is not fully tested, but we suppose it should work - msg := "ubuntu support is not fully tested, be careful" + msg := "Ubuntu support is not fully tested, be careful" result.Err = fmt.Errorf("%s (%s)", result.Msg, msg) result.Warn = true - if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 18.04 { - result.Err = fmt.Errorf("%s %s not supported, use version 18.04 or higher (%s)", + if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 20.04 { + result.Err = fmt.Errorf("%s %s not supported, use version 20.04 or higher (%s)", osInfo.Name, osInfo.Release, msg) result.Warn = false return result @@ -221,7 +240,7 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { case "openEuler": return result default: - result.Err = fmt.Errorf("os vendor %s not supported", osInfo.Vendor) + result.Err = fmt.Errorf("OS vendor %s not supported", osInfo.Vendor) return result } diff --git a/pkg/cluster/operation/download.go b/pkg/cluster/operation/download.go index a2dfb6a6f5..fe7f431df7 100644 --- a/pkg/cluster/operation/download.go +++ b/pkg/cluster/operation/download.go @@ -51,7 +51,7 @@ func Download(component, nodeOS, arch string, version string) error { if err := repo.DownloadComponent(component, version, targetPath); err != nil { return err } - } else { + } else if version != "nightly" { if err := repo.VerifyComponent(component, version, targetPath); err != nil { os.Remove(targetPath) } diff --git a/pkg/cluster/operation/upgrade.go b/pkg/cluster/operation/upgrade.go index 5c4c420858..3cace668ff 100644 --- a/pkg/cluster/operation/upgrade.go +++ b/pkg/cluster/operation/upgrade.go @@ -131,17 +131,19 @@ func Upgrade( if instance.IgnoreMonitorAgent() { noAgentHosts.Insert(instance.GetManageHost()) } + + // Usage within the switch statement switch component.Name() { - case spec.ComponentPD: - // defer PD leader to be upgraded after others - isLeader, err := instance.(*spec.PDInstance).IsLeader(ctx, topo, int(options.APITimeout), tlsCfg) + case spec.ComponentPD, spec.ComponentTSO, spec.ComponentScheduling: + // defer PD related leader/primary to be upgraded after others + isLeader, err := checkAndDeferPDLeader(ctx, topo, int(options.APITimeout), tlsCfg, instance) if err != nil { - logger.Warnf("cannot found pd leader, ignore: %s", err) + logger.Warnf("cannot found pd related leader/primary, ignore: %s, instance: %s", err, instance.ID()) return err } if isLeader { deferInstances = append(deferInstances, instance) - logger.Debugf("Deferred upgrading of PD leader %s", instance.ID()) + logger.Debugf("Upgrading deferred instance %s...", instance.ID()) continue } case spec.ComponentCDC: @@ -218,6 +220,22 @@ func Upgrade( return RestartMonitored(ctx, uniqueHosts.Slice(), noAgentHosts, topo.GetMonitoredOptions(), options.OptTimeout, systemdMode) } +// checkAndDeferPDLeader checks the PD related leader/primary instance's status and defers its upgrade if necessary. +func checkAndDeferPDLeader(ctx context.Context, topo spec.Topology, apiTimeout int, tlsCfg *tls.Config, instance spec.Instance) (isLeader bool, err error) { + switch instance.ComponentName() { + case spec.ComponentPD: + isLeader, err = instance.(*spec.PDInstance).IsLeader(ctx, topo, apiTimeout, tlsCfg) + case spec.ComponentScheduling: + isLeader, err = instance.(*spec.SchedulingInstance).IsPrimary(ctx, topo, tlsCfg) + case spec.ComponentTSO: + isLeader, err = instance.(*spec.TSOInstance).IsPrimary(ctx, topo, tlsCfg) + } + if err != nil { + return false, err + } + return isLeader, nil +} + func upgradeInstance( ctx context.Context, topo spec.Topology, diff --git a/pkg/cluster/spec/grafana_test.go b/pkg/cluster/spec/grafana_test.go index 802c179e82..d15ec09ccd 100644 --- a/pkg/cluster/spec/grafana_test.go +++ b/pkg/cluster/spec/grafana_test.go @@ -81,7 +81,7 @@ func TestMergeAdditionalGrafanaConf(t *testing.T) { ;enabled = false ;host = localhost:25 ;user = -;password = +password = ` + "`1#2`" + ` ;cert_file = ;key_file = ;skip_verify = false @@ -113,12 +113,12 @@ level = info expected := `# ################################### SMTP / Emailing ########################## [smtp] -enabled = true - ; enabled = false ; host = localhost:25 ; user = -; password = +password = ` + "`1#2`" + ` +enabled = true + ; cert_file = ; key_file = ; skip_verify = false diff --git a/pkg/cluster/spec/parse_topology_test.go b/pkg/cluster/spec/parse_topology_test.go index 51d0adfb74..4b544da329 100644 --- a/pkg/cluster/spec/parse_topology_test.go +++ b/pkg/cluster/spec/parse_topology_test.go @@ -540,6 +540,37 @@ tiflash_servers: }) } +func (s *topoSuite) TestMergeComponentVersions(c *check.C) { + // test component version overwrite + with2TempFile(` +component_versions: + tidb: v8.0.0 + tikv: v8.0.0 +tidb_servers: + - host: 172.16.5.139 +`, ` +component_versions: + tikv: v8.1.0 + pd: v8.0.0 +tidb_servers: + - host: 172.16.5.134 +`, func(base, scale string) { + baseTopo := Specification{} + c.Assert(ParseTopologyYaml(base, &baseTopo), check.IsNil) + + scaleTopo := baseTopo.NewPart() + c.Assert(ParseTopologyYaml(scale, scaleTopo), check.IsNil) + + mergedTopo := baseTopo.MergeTopo(scaleTopo) + c.Assert(mergedTopo.Validate(), check.IsNil) + + c.Assert(scaleTopo.(*Specification).ComponentVersions, check.Equals, mergedTopo.(*Specification).ComponentVersions) + c.Assert(scaleTopo.(*Specification).ComponentVersions.TiDB, check.Equals, "v8.0.0") + c.Assert(scaleTopo.(*Specification).ComponentVersions.TiKV, check.Equals, "v8.1.0") + c.Assert(scaleTopo.(*Specification).ComponentVersions.PD, check.Equals, "v8.0.0") + }) +} + func (s *topoSuite) TestFixRelativePath(c *check.C) { // base test topo := Specification{ diff --git a/pkg/cluster/spec/scheduling.go b/pkg/cluster/spec/scheduling.go index efbe7def00..31957570d3 100644 --- a/pkg/cluster/spec/scheduling.go +++ b/pkg/cluster/spec/scheduling.go @@ -21,30 +21,36 @@ import ( "strings" "time" + "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/utils" ) +var schedulingService = "scheduling" + // SchedulingSpec represents the scheduling topology specification in topology.yaml type SchedulingSpec struct { - Host string `yaml:"host"` - ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` - ListenHost string `yaml:"listen_host,omitempty"` - AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` - SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` - IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` - Port int `yaml:"port" default:"3379"` - DeployDir string `yaml:"deploy_dir,omitempty"` - DataDir string `yaml:"data_dir,omitempty"` - LogDir string `yaml:"log_dir,omitempty"` - Source string `yaml:"source,omitempty" validate:"source:editable"` - NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` - Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` - Arch string `yaml:"arch,omitempty"` - OS string `yaml:"os,omitempty"` + Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` + ListenHost string `yaml:"listen_host,omitempty"` + AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` + SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` + IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` + // Use Name to get the name with a default value if it's empty. + Name string `yaml:"name,omitempty"` + Port int `yaml:"port" default:"3379"` + DeployDir string `yaml:"deploy_dir,omitempty"` + DataDir string `yaml:"data_dir,omitempty"` + LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` + NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` + Arch string `yaml:"arch,omitempty"` + OS string `yaml:"os,omitempty"` } // Status queries current status of the instance @@ -63,7 +69,7 @@ func (s *SchedulingSpec) Status(ctx context.Context, timeout time.Duration, tlsC return "Down" } - primary, err := pc.GetServicePrimary("scheduling") + primary, err := pc.GetServicePrimary(schedulingService) if err != nil { return "ERR" } @@ -200,7 +206,6 @@ func (c *SchedulingComponent) Instances() []Instance { // SchedulingInstance represent the scheduling instance type SchedulingInstance struct { - Name string BaseInstance topo Topology } @@ -229,6 +234,7 @@ func (i *SchedulingInstance) InitConfig( pds = append(pds, pdspec.GetAdvertiseClientURL(enableTLS)) } cfg := &scripts.SchedulingScript{ + Name: spec.Name, ListenURL: fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(i.GetListenHost(), spec.Port)), AdvertiseListenURL: spec.GetAdvertiseListenURL(enableTLS), BackendEndpoints: strings.Join(pds, ","), @@ -237,6 +243,9 @@ func (i *SchedulingInstance) InitConfig( LogDir: paths.Log, NumaNode: spec.NumaNode, } + if !tidbver.PDSupportMicroServicesWithName(version) { + cfg.Name = "" + } fp := filepath.Join(paths.Cache, fmt.Sprintf("run_scheduling_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { @@ -303,6 +312,27 @@ func (i *SchedulingInstance) setTLSConfig(ctx context.Context, enableTLS bool, c return configs, nil } +// IsPrimary checks if the instance is primary +func (i *SchedulingInstance) IsPrimary(ctx context.Context, topo Topology, tlsCfg *tls.Config) (bool, error) { + tidbTopo, ok := topo.(*Specification) + if !ok { + panic("topo should be type of tidb topology") + } + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDListWithManageHost(), time.Second*5, tlsCfg) + primary, err := pdClient.GetServicePrimary(schedulingService) + if err != nil { + return false, errors.Annotatef(err, "failed to get Scheduling primary %s", i.GetHost()) + } + + spec := i.InstanceSpec.(*SchedulingSpec) + enableTLS := false + if tlsCfg != nil { + enableTLS = true + } + + return primary == spec.GetAdvertiseListenURL(enableTLS), nil +} + // ScaleConfig deploy temporary config on scaling func (i *SchedulingInstance) ScaleConfig( ctx context.Context, diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index ffef492dda..6f299bb4c6 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -270,9 +270,10 @@ type UpgradableMetadata interface { // NewPart implements ScaleOutTopology interface. func (s *Specification) NewPart() Topology { return &Specification{ - GlobalOptions: s.GlobalOptions, - MonitoredOptions: s.MonitoredOptions, - ServerConfigs: s.ServerConfigs, + GlobalOptions: s.GlobalOptions, + MonitoredOptions: s.MonitoredOptions, + ServerConfigs: s.ServerConfigs, + ComponentVersions: s.ComponentVersions, } } @@ -518,8 +519,8 @@ func (s *Specification) AdjustByVersion(clusterVersion string) { } } -// GetDashboardAddress returns the cluster's dashboard addr -func (s *Specification) GetDashboardAddress(ctx context.Context, tlsCfg *tls.Config, timeout time.Duration, pdList ...string) (string, error) { +// GetPDDashboardAddress returns the cluster's dashboard addr +func (s *Specification) GetPDDashboardAddress(ctx context.Context, tlsCfg *tls.Config, timeout time.Duration, pdList ...string) (string, error) { if timeout < time.Second { timeout = statusQueryTimeout } @@ -679,10 +680,20 @@ func setCustomDefaults(globalOptions *GlobalOptions, field reflect.Value) error } field.Field(j).Set(reflect.ValueOf(globalOptions.SSHPort)) case "Name": + // Only PD related components have `Name` field, if field.Field(j).String() != "" { continue } host := reflect.Indirect(field).FieldByName("Host").String() + // `TSO` and `Scheduling` components use `Port` filed + if reflect.Indirect(field).FieldByName("Port").IsValid() { + port := reflect.Indirect(field).FieldByName("Port").Int() + // field.String() is + role := strings.Split(strings.Split(field.Type().String(), ".")[1], "Spec")[0] + component := strings.ToLower(role) + field.Field(j).Set(reflect.ValueOf(fmt.Sprintf("%s-%s-%d", component, host, port))) + continue + } clientPort := reflect.Indirect(field).FieldByName("ClientPort").Int() field.Field(j).Set(reflect.ValueOf(fmt.Sprintf("pd-%s-%d", host, clientPort))) case "DataDir": diff --git a/pkg/cluster/spec/tso.go b/pkg/cluster/spec/tso.go index a84069f6ee..3b02f0630e 100644 --- a/pkg/cluster/spec/tso.go +++ b/pkg/cluster/spec/tso.go @@ -21,30 +21,36 @@ import ( "strings" "time" + "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/utils" ) +var tsoService = "tso" + // TSOSpec represents the TSO topology specification in topology.yaml type TSOSpec struct { - Host string `yaml:"host"` - ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` - ListenHost string `yaml:"listen_host,omitempty"` - AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` - SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` - IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` - Port int `yaml:"port" default:"3379"` - DeployDir string `yaml:"deploy_dir,omitempty"` - DataDir string `yaml:"data_dir,omitempty"` - LogDir string `yaml:"log_dir,omitempty"` - Source string `yaml:"source,omitempty" validate:"source:editable"` - NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` - Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` - Arch string `yaml:"arch,omitempty"` - OS string `yaml:"os,omitempty"` + Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` + ListenHost string `yaml:"listen_host,omitempty"` + AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` + SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` + IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` + // Use Name to get the name with a default value if it's empty. + Name string `yaml:"name,omitempty"` + Port int `yaml:"port" default:"3379"` + DeployDir string `yaml:"deploy_dir,omitempty"` + DataDir string `yaml:"data_dir,omitempty"` + LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` + NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` + Arch string `yaml:"arch,omitempty"` + OS string `yaml:"os,omitempty"` } // Status queries current status of the instance @@ -63,7 +69,7 @@ func (s *TSOSpec) Status(ctx context.Context, timeout time.Duration, tlsCfg *tls return "Down" } - primary, err := pc.GetServicePrimary("tso") + primary, err := pc.GetServicePrimary(tsoService) if err != nil { return "ERR" } @@ -200,7 +206,6 @@ func (c *TSOComponent) Instances() []Instance { // TSOInstance represent the TSO instance type TSOInstance struct { - Name string BaseInstance topo Topology } @@ -229,6 +234,7 @@ func (i *TSOInstance) InitConfig( pds = append(pds, pdspec.GetAdvertiseClientURL(enableTLS)) } cfg := &scripts.TSOScript{ + Name: spec.Name, ListenURL: fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(i.GetListenHost(), spec.Port)), AdvertiseListenURL: spec.GetAdvertiseListenURL(enableTLS), BackendEndpoints: strings.Join(pds, ","), @@ -237,6 +243,9 @@ func (i *TSOInstance) InitConfig( LogDir: paths.Log, NumaNode: spec.NumaNode, } + if !tidbver.PDSupportMicroServicesWithName(version) { + cfg.Name = "" + } fp := filepath.Join(paths.Cache, fmt.Sprintf("run_tso_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { @@ -303,6 +312,27 @@ func (i *TSOInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs return configs, nil } +// IsPrimary checks if the instance is primary +func (i *TSOInstance) IsPrimary(ctx context.Context, topo Topology, tlsCfg *tls.Config) (bool, error) { + tidbTopo, ok := topo.(*Specification) + if !ok { + panic("topo should be type of tidb topology") + } + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDListWithManageHost(), time.Second*5, tlsCfg) + primary, err := pdClient.GetServicePrimary(tsoService) + if err != nil { + return false, errors.Annotatef(err, "failed to get TSO primary %s", i.GetHost()) + } + + spec := i.InstanceSpec.(*TSOSpec) + enableTLS := false + if tlsCfg != nil { + enableTLS = true + } + + return primary == spec.GetAdvertiseListenURL(enableTLS), nil +} + // ScaleConfig deploy temporary config on scaling func (i *TSOInstance) ScaleConfig( ctx context.Context, diff --git a/pkg/cluster/spec/validate.go b/pkg/cluster/spec/validate.go index 99384cbec7..ce73682c01 100644 --- a/pkg/cluster/spec/validate.go +++ b/pkg/cluster/spec/validate.go @@ -984,6 +984,38 @@ func (s *Specification) validatePDNames() error { return nil } +func (s *Specification) validateTSONames() error { + // check tso server name + tsoNames := set.NewStringSet() + for _, tso := range s.TSOServers { + if tso.Name == "" { + continue + } + + if tsoNames.Exist(tso.Name) { + return errors.Errorf("component tso_servers.name is not supported duplicated, the name %s is duplicated", tso.Name) + } + tsoNames.Insert(tso.Name) + } + return nil +} + +func (s *Specification) validateSchedulingNames() error { + // check scheduling server name + schedulingNames := set.NewStringSet() + for _, scheduling := range s.SchedulingServers { + if scheduling.Name == "" { + continue + } + + if schedulingNames.Exist(scheduling.Name) { + return errors.Errorf("component scheduling_servers.name is not supported duplicated, the name %s is duplicated", scheduling.Name) + } + schedulingNames.Insert(scheduling.Name) + } + return nil +} + func (s *Specification) validateTiFlashConfigs() error { c := FindComponent(s, ComponentTiFlash) for _, ins := range c.Instances() { @@ -1063,6 +1095,8 @@ func (s *Specification) Validate() error { s.dirConflictsDetect, s.validateUserGroup, s.validatePDNames, + s.validateTSONames, + s.validateSchedulingNames, s.validateTiSparkSpec, s.validateTiFlashConfigs, s.validateMonitorAgent, diff --git a/pkg/cluster/template/install/local_install.sh.go b/pkg/cluster/template/install/local_install.sh.go index b08ee6bb0f..7030329434 100644 --- a/pkg/cluster/template/install/local_install.sh.go +++ b/pkg/cluster/template/install/local_install.sh.go @@ -81,16 +81,17 @@ fi chmod 755 "$bin_dir/tiup" -# telemetry is not needed for offline installations -"$bin_dir/tiup" telemetry disable - # set mirror to the local path -"$bin_dir/tiup" mirror set ${script_dir} +"$bin_dir/tiup" mirror set ${script_dir} --silent bold=$(tput bold 2>/dev/null) +green=$(tput setaf 2 2>/dev/null) +cyan=$(tput setaf 6 2>/dev/null) sgr0=$(tput sgr0 2>/dev/null) -# Reference: https://stackoverflow.com/questions/14637979/how-to-permanently-set-path-on-linux-unix +echo + +# Refrence: https://stackoverflow.com/questions/14637979/how-to-permanently-set-path-on-linux-unix shell=$(echo $SHELL | awk 'BEGIN {FS="/";} { print $NF }') echo "Detected shell: ${bold}$shell${sgr0}" if [ -f "${HOME}/.${shell}_profile" ]; then @@ -103,18 +104,23 @@ else PROFILE=${HOME}/.profile fi echo "Shell profile: ${bold}$PROFILE${sgr0}" - +echo +echo "${bold}${green}✔ ${sgr0}Installed in ${bold}$bin_dir/tiup${sgr0}" case :$PATH: in - *:$bin_dir:*) : "PATH already contains $bin_dir" ;; - *) printf 'export PATH=%s:$PATH\n' "$bin_dir" >> "$PROFILE" - echo "$PROFILE has been modified to to add tiup to PATH" - echo "open a new terminal or ${bold}source ${PROFILE}${sgr0} to use it" + *:$bin_dir:*) echo "${bold}${green}✔ ${sgr0}tiup PATH is already set, skip" ;; + *) printf '\nexport PATH=%s:$PATH\n' "$bin_dir" >> "$PROFILE" + echo "${bold}${green}✔ ${sgr0}Added tiup PATH into ${bold}${shell}${sgr0} profile" ;; esac +echo +echo "${bold}tiup is installed now${sgr0} 🎉" +echo +echo Next step: +echo +echo " 1: To make PATH change effective, restart your shell or execute:" +echo " ${bold}${cyan}source ${PROFILE}${sgr0}" +echo +echo " 2: Start a local TiDB for development:" +echo " ${bold}${cyan}tiup playground${sgr0}" -echo "Installed path: ${bold}$bin_dir/tiup${sgr0}" -echo "===============================================" -echo "1. ${bold}source ${PROFILE}${sgr0}" -echo "2. Have a try: ${bold}tiup playground${sgr0}" -echo "===============================================" ` diff --git a/pkg/cluster/template/scripts/pdms_test.go b/pkg/cluster/template/scripts/pdms_test.go new file mode 100644 index 0000000000..414f693f1f --- /dev/null +++ b/pkg/cluster/template/scripts/pdms_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scripts + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestScheduling(t *testing.T) { + assert := require.New(t) + conf, err := os.CreateTemp("", "scheduling.conf") + assert.Nil(err) + defer os.Remove(conf.Name()) + + cfg := &SchedulingScript{ + Name: "scheduling-0", + ListenURL: "127.0.0.1", + AdvertiseListenURL: "127.0.0.2", + BackendEndpoints: "127.0.0.3", + DeployDir: "/deploy", + DataDir: "/data", + LogDir: "/log", + } + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err := os.ReadFile(conf.Name()) + assert.Nil(err) + assert.True(strings.Contains(string(content), "--name")) + + cfg.Name = "" + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err = os.ReadFile(conf.Name()) + assert.Nil(err) + assert.False(strings.Contains(string(content), "--name")) +} + +func TestTSO(t *testing.T) { + assert := require.New(t) + conf, err := os.CreateTemp("", "tso.conf") + assert.Nil(err) + defer os.Remove(conf.Name()) + + cfg := &TSOScript{ + Name: "tso-0", + ListenURL: "127.0.0.1", + AdvertiseListenURL: "127.0.0.2", + BackendEndpoints: "127.0.0.3", + DeployDir: "/deploy", + DataDir: "/data", + LogDir: "/log", + } + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err := os.ReadFile(conf.Name()) + assert.Nil(err) + assert.True(strings.Contains(string(content), "--name")) + + cfg.Name = "" + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err = os.ReadFile(conf.Name()) + assert.Nil(err) + assert.False(strings.Contains(string(content), "--name")) +} diff --git a/pkg/cluster/template/scripts/scheduling.go b/pkg/cluster/template/scripts/scheduling.go index 6167d9336e..76142485a2 100644 --- a/pkg/cluster/template/scripts/scheduling.go +++ b/pkg/cluster/template/scripts/scheduling.go @@ -24,6 +24,7 @@ import ( // SchedulingScript represent the data to generate scheduling config type SchedulingScript struct { + Name string ListenURL string AdvertiseListenURL string BackendEndpoints string diff --git a/pkg/cluster/template/scripts/tso.go b/pkg/cluster/template/scripts/tso.go index 0197b82c38..91c3bfe1d0 100644 --- a/pkg/cluster/template/scripts/tso.go +++ b/pkg/cluster/template/scripts/tso.go @@ -24,6 +24,7 @@ import ( // TSOScript represent the data to generate tso config type TSOScript struct { + Name string ListenURL string AdvertiseListenURL string BackendEndpoints string diff --git a/pkg/crypto/rand/rand.go b/pkg/crypto/rand/rand.go index ebf77edbe6..aad1108dbb 100644 --- a/pkg/crypto/rand/rand.go +++ b/pkg/crypto/rand/rand.go @@ -14,42 +14,45 @@ package rand import ( - cr "crypto/rand" - "encoding/binary" - "fmt" - "math/rand" + "crypto/rand" + "io" + "math/big" ) -var ( - // Reader is a global random number source - Reader *rand.Rand -) +// rand provides a simple wrap of "crypto/rand". -func init() { - src := make([]byte, 8) - if _, err := cr.Read(src); err != nil { - panic(fmt.Sprintf("initial random: %s", err.Error())) - } - seed := binary.BigEndian.Uint64(src) - Reader = rand.New(rand.NewSource(int64(seed))) +// Reader is a global random number source +var Reader io.Reader = &cryptoRandReader{} + +type cryptoRandReader struct{} + +func (c *cryptoRandReader) Read(b []byte) (int, error) { + return rand.Read(b) } -// Int wraps Rand.Int +// Int wraps Int63n func Int() int { - return Reader.Int() + val := Int63n(int64(int(^uint(0) >> 1))) + return int(val) } -// Intn wraps Rand.Intn +// Intn wraps Int63n func Intn(n int) int { - return Reader.Intn(n) + if n <= 0 { + panic("argument to Intn must be positive") + } + val := Int63n(int64(n)) + return int(val) } -// Int63n wraps Rand.Int63n +// Int63n wraps rand.Int func Int63n(n int64) int64 { - return Reader.Int63n(n) -} - -// Read wraps Rand.Read -func Read(b []byte) (int, error) { - return Reader.Read(b) + if n <= 0 { + panic("argument to Int63n must be positive") + } + val, err := rand.Int(rand.Reader, big.NewInt(n)) + if err != nil { + panic(err) + } + return val.Int64() } diff --git a/pkg/localdata/profile.go b/pkg/localdata/profile.go index f9a28816ab..d66fc92795 100644 --- a/pkg/localdata/profile.go +++ b/pkg/localdata/profile.go @@ -275,9 +275,9 @@ func (p *Profile) ResetMirror(addr, root string) error { // Only cache remote mirror if strings.HasPrefix(addr, "http") && root != localRoot { - if strings.HasPrefix(root, "http") { - fmt.Printf("WARN: adding root certificate via internet: %s\n", root) - fmt.Printf("You can revoke this by remove %s\n", localRoot) + if strings.HasPrefix(root, "http") && !strings.HasPrefix(root, "https") { + fmt.Printf("WARN: Trusting component distribution key via insecure Internet: %s\n", root) + fmt.Printf(" To revoke TiUP's trust, remove this file: %s\n", localRoot) } _ = utils.Copy(p.Path("bin", "root.json"), localRoot) } diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index e83df39df6..63b1dd6384 100644 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -49,11 +49,7 @@ func MaybeStartProxy( return err } - httpPort, err := utils.GetFreePort("127.0.0.1", 12345) - if err != nil { - return err - } - + httpPort := utils.MustGetFreePort("127.0.0.1", 12345, 0) addr := fmt.Sprintf("127.0.0.1:%d", httpPort) // TODO: Using environment variables to share data may not be a good idea diff --git a/pkg/proxy/tcp_proxy.go b/pkg/proxy/tcp_proxy.go index d758c06506..6250e9fd7f 100644 --- a/pkg/proxy/tcp_proxy.go +++ b/pkg/proxy/tcp_proxy.go @@ -66,11 +66,7 @@ func NewTCPProxy( p.config.Password = password } - port, err := utils.GetFreePort("127.0.0.1", 22345) - if err != nil { - logger.Errorf("get free port error: %v", err) - return nil - } + port = utils.MustGetFreePort("127.0.0.1", 22345, 0) p.endpoint = fmt.Sprintf("127.0.0.1:%d", port) listener, err := net.Listen("tcp", p.endpoint) diff --git a/pkg/repository/testdata/manifests/root.json b/pkg/repository/testdata/manifests/root.json index ca16fcc0cb..0af4b4c619 100644 --- a/pkg/repository/testdata/manifests/root.json +++ b/pkg/repository/testdata/manifests/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kJdviRCj2xr6Mqb8maOJZUnBgHJqLI5V2MqhQL9mxqsHFE/6z3bmT1OIpaG31b0COhIg0gtR60Y/rUu1YRTs8UpUJa+9bHVcLxrg6PuewzT34ZjtsGpQ8PDhe2j99EULN85rq7jV+HRwdZLHVZXI1P25I0wy1AGtprVo13ZB4h5CBy4ZEUlmae0QKbH+Ej9ouVaeBlyw1JHLew8I1eWEj7x9WYMkO6spqDtPhrIYjqxnh9zUBAvMNQzv3ysu0KM+D4Pr6kALRE9G3AMmbCEzhy+C+r/FT7egNv28kJvxoQjgfEMABYLgLU/Gp7qRVbEbthH4XI8IohmiCH6NavQKBA=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cQAJkEEpUdauI7TN2nrEF/wZbPKa/Rhu/xEDZgm5tBAc0LKhD48aLH0VdxapytSf1IWk+MatEK/vVMA2sppqrJTGyJ5Ue+JEUe+JhcTAFW0sUzTHUgR5ARTFqDR0RnKysPO/a9BMZsJfyEfS2j2Nlx7iE/2PeMVc2GhhSEY+cHE000ZFXxaxuWYjYzd35LeLQotMWmtsEx0JYrqHgXzMU0yJ0QikE1JUd8uQ4LMvd53iPIEuHxKyq87oIMfvx+gKhbGvgQx3XMYI54qMP+fumPVYQ3alYX5FB+vFX2MFqzl7GxAQJCd57xsanRhKSAkG2LKvEUZuCfTtqiwqb13Lyg=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"cPqcr9zhryFMREpIm81u+vAy1anzJFEFwWk6gCSRNW8lZuGQPgF66qc09YGHe6wGFHmlAm7CTFKh561W8zJfi6IbGS7wUArkaNV5fgPkxg2FDGCC5tHs8we3fgyv+kgr6yzGwQVHl6v4J8S5bkK1MM7GbMGAP6dmf774f6ysLOkjGurAg7UwrZey8/ZQKCjzysnAxyRC+/4QhUfAC5OPs2hI+BUjPDTET3f1Icgq0f9CZfUZ4nW1utMd4/sbQKowNVfZ8j62r8awUuOGmmJbuqLs6ifkfKCXw5ZfrT9G8ySxbI8CshWd54Ck6c7WsyRjY4Y9TB+Oh3pJ+ieE91HJKg=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"lnfUXTabAURE8BKzbGBJ4oXtDYbpXl1hsEtzOfjJwBae+c9qMusYMR9aU3ZOm2hlAasUXf+bfIy5NoDZMuDC8roYD3UHokC2qrym1h3VgfVl1aMlWDAgtIhNQuHVbKGyPBDMW+O0gDIV2RPwTwcedZHxLbakMqgHaSI1SPFu4ryoLbOsBr2wxQSVKUXRhcvQFvgRJS884TfIYmY3v4EIHvSfHrnPmDmMFrpMUCVzseDPMG/FEWamzjk0GnRAZiwv5NVjbtIAtIUxBASZRWQr1h176k03jwPodT/iynNTNMy/2WHMwUSrVxeFB+aXI9+Pbl/OHw3UAz2qp3kS2ny0aQ=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"QilsuGt2x4NjB8uTQeFaUb8sfRwhwcEKGqBBAXLPtgCLm7GXxRb4RXO8nPQ0o5Kg1nZp7t899Q5nMzOO6Qc5ng3vTOwDW2cDSVhyllPKJAzgJC5uwHQdQi9y3vSiNb4j2mpkQbyfu9va5yUiAZzSoJHuRv+aperJHvv+Ev/kxqlvy/4TtXtMPebG2qc08K/WdXBA0S++oWhsC8J5mFgBJBFXFJ6ewLlRJK7DmZKEG+0vtaOeCLOsRnKFSQ3rfrMYdBnGU32+NtlmTU4rjpZ/HdSwgi4K8tDVoo1CE9EwvLvZ1oKpHyRiZWmviRCy81WVB/kpYgfSR4u1CQ02CgZVkQ=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file +{"signatures":[{"keyid":"18007f60ccfdf9fa5bc79b09464880c054ce34289e97ad93696b85cc43aed314","sig":"fNbhxgGrR/opxc1gaYZtyaDYAYfpq52eGB8v5lT8s1OoboOCTMC975DOYH1P3cC43qCtwwYOtJviBKG7SQ3lxzIR3wSaEqnJMtZx6VFsn7k8znFKRQBU0Tw5veHtjRxtX3SdudQ+qQPvgVBbTwxWRLVZiPH5D7aFzo7ylBauOlrx+DdhURM29JmU/fcXUcLsjFxp/A6oguQWoUWiGDfT7LqdBUxKDNU0wk+AUX2UEeGavIz/L0Ja6/VVKPwzI2C9iPYYezuUlKKi6PR9XDX/X1+CGuEHlpYeITY/Ipgy3zhY2svXHCvgvXBAx5bqJPyReNDaQAx9QnBzrij3k1Vpuw=="},{"keyid":"ef644d810a1f1dcce7078ae5b2821cba346a2eac0a371e56feea9e07a5eade37","sig":"ldOj6IMwRc/ANElR9ZHfAInFYJfZ+eqzcdehfdYPU4GgpQBK8vKe/Vi7oYyaAWM+oRi7KlIUPWSsnxd5aB0J0hznERPtQzoBg/pNVX5f2ls3EU+21YdSJ7TrD6uud+oqo4tumkjmE7SYSIY4064colPeevob+uJSt2w4MgKa3LCBSsE+vltBo/m5GYz2vKifOL1j2eXRtcBr8PiAasANfZkm+uAlBgKCt+ZSZawGQ3LyuxSimvdTHd40og2ugG03GBMTwOP4igubKodFI+gzNvFXy1BfD8wd7Aa8GQ25EAH6NUARDmyunlW7s6qG+rWivFouvWC3iCTErKYDP1XP0w=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"V+ab8aZ7CQtDqZLDlFEB2V6bL6EGpjsoeFNimMftDXpjAel81JvR/F6ta1f0daatpHky+Zefh5jnVNE2CbezM5lymIhfXKbGBgPY6UrXY5NmMEjv4miVg/t0trkzNqHnOsX+4jvz81dbu9BUhvCEP96wh3O+SltIhToPsleTJuAOSSU9eptS5wCYTQqMjClrpurivs8XRgXeryRorGjWCs8DxhHZqe/hvivcHp9YefvkLw9VQzjZlaH3SCHShLlkaZpn2qssCmzuSt4PLD97VZW89QD/kOqlbHVCqZNzzHp1i6UG6hccN/hnhiYo1v4hOgeX6QT5R8/rlwF8M/RZeQ=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"TVbpxEpaobzt/ODTm+aC9YlhPRvr8NaEREJ4sOoGZCllNxeDaAeSB16VuzmoOfE7esDZBwPuZ8jN1HgdSz33H85/2KRg8EDllCo9fKgKbYhIxr4WgJDoqf3y9ejrjMHRJW8ReCntpobeajuKTxOvgZEvi9wJJ9DagDk9V2u+62jQpMNJdswa6tRH/WsVjNFQ5FMmuf4UasuwhkSupZHjwtFx0B63Jc//hVC/mgCdP30oChq7dkI63tX7eJDu4L07V/MOUl2LBemX5z5OWXDFq5AykIn6re/uzna92EPINh3waa+0Wr0VK70Znj1cDBxb1q6aijIMVpnuXA+P5BeB1A=="},{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kzFOjJlBSJtADdze6nmpXe6fgnz46rXBWgfACInRldm1pr7sO/Exugwa4qBn0yp2a0N+mDhXvxEi+CuYYOqSNJe5z3TjqiBpTBo2fvOsb6HxNV3dhb24D465rzBFEqi8VxU/KoU/nhlCcGUev+knYJ+oYySjrfJpnxSXmoWYYJiCM/lNcuySqUzeH2WO69pKGDwtEmCXZaRuKwXt3B2HUjLny7aRmLV9Z28pCGobmpEAUIDM2/sU9BWicWSzTD8OAEkX1WPCIAm6I2Q/T0T7ImDbeL8nX4ctzbUuW7uoXM7aM9eq26kaYJ+mqwqQMBNuMYGDq5tShzTsRk9KNpkhrw=="}],"signed":{"_type":"root","expires":"2025-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"18007f60ccfdf9fa5bc79b09464880c054ce34289e97ad93696b85cc43aed314":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4DYlVfoIQTlyJij0ynjh\njqUkayqXX5c9VXw1Ud3mWCOdThy6V0bmsohgSBeHrfVroSCfsAc5VCUlaSteZeFl\nQEZxpRWDCmSYGslOQZqe2cJi5aqyQOYeU7JLjlfAausLCR9636SfEvQoaCEuGsUI\n67yCVWW2oQ756egUNmOrOSd7Qh6IGuuj9FQb9vExPXTxQw7j95ENOsc1V2lAXCEG\nS1+Nh4NIKdpLOXAohbcpq/HLjddmEAj2GXHo+asITlHCVUQvf574Vh5yLkFWnqj0\nviyRq0jJa9P+qA2oy80a3dk3FBCPu0sov6GfUIC+NtkDfjOkKfluBF9WapqR9wt0\noQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"ef644d810a1f1dcce7078ae5b2821cba346a2eac0a371e56feea9e07a5eade37":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqsL5sV9dhPqnkc3dU2xH\nVRPVuH1bebET64bJya96IXjR3Um/IbIikmIpAL8KbY35h44hR4nNwUQZcQggo854\n5SxDi5LiAkMqdr9uq5mXp7sZXb0HcuHX97BqTUvTvr+t05KaON81ikdVGyRw+Qus\nFFXZO2Pj0w0I4QD87nISAuK0wQJhD8robDzO+Qf2K5cHXjEu5DGNc+wq66pJWCwt\nDl2BAvkF86Y3kZVuEQ6zp5PPQh0l++0PtzY/NNNHiLm7JUSlmpXyis7f+FaCEGl0\n4JWs5ImJg1XjUo2AsSnlFZ3adrPJ4NHFo64ui0/JsEAhn1TBWLL4AhT9kVIBMXI4\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":5}} \ No newline at end of file diff --git a/pkg/repository/testdata/polluted/bin/root.json b/pkg/repository/testdata/polluted/bin/root.json index ca16fcc0cb..0af4b4c619 100644 --- a/pkg/repository/testdata/polluted/bin/root.json +++ b/pkg/repository/testdata/polluted/bin/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kJdviRCj2xr6Mqb8maOJZUnBgHJqLI5V2MqhQL9mxqsHFE/6z3bmT1OIpaG31b0COhIg0gtR60Y/rUu1YRTs8UpUJa+9bHVcLxrg6PuewzT34ZjtsGpQ8PDhe2j99EULN85rq7jV+HRwdZLHVZXI1P25I0wy1AGtprVo13ZB4h5CBy4ZEUlmae0QKbH+Ej9ouVaeBlyw1JHLew8I1eWEj7x9WYMkO6spqDtPhrIYjqxnh9zUBAvMNQzv3ysu0KM+D4Pr6kALRE9G3AMmbCEzhy+C+r/FT7egNv28kJvxoQjgfEMABYLgLU/Gp7qRVbEbthH4XI8IohmiCH6NavQKBA=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cQAJkEEpUdauI7TN2nrEF/wZbPKa/Rhu/xEDZgm5tBAc0LKhD48aLH0VdxapytSf1IWk+MatEK/vVMA2sppqrJTGyJ5Ue+JEUe+JhcTAFW0sUzTHUgR5ARTFqDR0RnKysPO/a9BMZsJfyEfS2j2Nlx7iE/2PeMVc2GhhSEY+cHE000ZFXxaxuWYjYzd35LeLQotMWmtsEx0JYrqHgXzMU0yJ0QikE1JUd8uQ4LMvd53iPIEuHxKyq87oIMfvx+gKhbGvgQx3XMYI54qMP+fumPVYQ3alYX5FB+vFX2MFqzl7GxAQJCd57xsanRhKSAkG2LKvEUZuCfTtqiwqb13Lyg=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"cPqcr9zhryFMREpIm81u+vAy1anzJFEFwWk6gCSRNW8lZuGQPgF66qc09YGHe6wGFHmlAm7CTFKh561W8zJfi6IbGS7wUArkaNV5fgPkxg2FDGCC5tHs8we3fgyv+kgr6yzGwQVHl6v4J8S5bkK1MM7GbMGAP6dmf774f6ysLOkjGurAg7UwrZey8/ZQKCjzysnAxyRC+/4QhUfAC5OPs2hI+BUjPDTET3f1Icgq0f9CZfUZ4nW1utMd4/sbQKowNVfZ8j62r8awUuOGmmJbuqLs6ifkfKCXw5ZfrT9G8ySxbI8CshWd54Ck6c7WsyRjY4Y9TB+Oh3pJ+ieE91HJKg=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"lnfUXTabAURE8BKzbGBJ4oXtDYbpXl1hsEtzOfjJwBae+c9qMusYMR9aU3ZOm2hlAasUXf+bfIy5NoDZMuDC8roYD3UHokC2qrym1h3VgfVl1aMlWDAgtIhNQuHVbKGyPBDMW+O0gDIV2RPwTwcedZHxLbakMqgHaSI1SPFu4ryoLbOsBr2wxQSVKUXRhcvQFvgRJS884TfIYmY3v4EIHvSfHrnPmDmMFrpMUCVzseDPMG/FEWamzjk0GnRAZiwv5NVjbtIAtIUxBASZRWQr1h176k03jwPodT/iynNTNMy/2WHMwUSrVxeFB+aXI9+Pbl/OHw3UAz2qp3kS2ny0aQ=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"QilsuGt2x4NjB8uTQeFaUb8sfRwhwcEKGqBBAXLPtgCLm7GXxRb4RXO8nPQ0o5Kg1nZp7t899Q5nMzOO6Qc5ng3vTOwDW2cDSVhyllPKJAzgJC5uwHQdQi9y3vSiNb4j2mpkQbyfu9va5yUiAZzSoJHuRv+aperJHvv+Ev/kxqlvy/4TtXtMPebG2qc08K/WdXBA0S++oWhsC8J5mFgBJBFXFJ6ewLlRJK7DmZKEG+0vtaOeCLOsRnKFSQ3rfrMYdBnGU32+NtlmTU4rjpZ/HdSwgi4K8tDVoo1CE9EwvLvZ1oKpHyRiZWmviRCy81WVB/kpYgfSR4u1CQ02CgZVkQ=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file +{"signatures":[{"keyid":"18007f60ccfdf9fa5bc79b09464880c054ce34289e97ad93696b85cc43aed314","sig":"fNbhxgGrR/opxc1gaYZtyaDYAYfpq52eGB8v5lT8s1OoboOCTMC975DOYH1P3cC43qCtwwYOtJviBKG7SQ3lxzIR3wSaEqnJMtZx6VFsn7k8znFKRQBU0Tw5veHtjRxtX3SdudQ+qQPvgVBbTwxWRLVZiPH5D7aFzo7ylBauOlrx+DdhURM29JmU/fcXUcLsjFxp/A6oguQWoUWiGDfT7LqdBUxKDNU0wk+AUX2UEeGavIz/L0Ja6/VVKPwzI2C9iPYYezuUlKKi6PR9XDX/X1+CGuEHlpYeITY/Ipgy3zhY2svXHCvgvXBAx5bqJPyReNDaQAx9QnBzrij3k1Vpuw=="},{"keyid":"ef644d810a1f1dcce7078ae5b2821cba346a2eac0a371e56feea9e07a5eade37","sig":"ldOj6IMwRc/ANElR9ZHfAInFYJfZ+eqzcdehfdYPU4GgpQBK8vKe/Vi7oYyaAWM+oRi7KlIUPWSsnxd5aB0J0hznERPtQzoBg/pNVX5f2ls3EU+21YdSJ7TrD6uud+oqo4tumkjmE7SYSIY4064colPeevob+uJSt2w4MgKa3LCBSsE+vltBo/m5GYz2vKifOL1j2eXRtcBr8PiAasANfZkm+uAlBgKCt+ZSZawGQ3LyuxSimvdTHd40og2ugG03GBMTwOP4igubKodFI+gzNvFXy1BfD8wd7Aa8GQ25EAH6NUARDmyunlW7s6qG+rWivFouvWC3iCTErKYDP1XP0w=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"V+ab8aZ7CQtDqZLDlFEB2V6bL6EGpjsoeFNimMftDXpjAel81JvR/F6ta1f0daatpHky+Zefh5jnVNE2CbezM5lymIhfXKbGBgPY6UrXY5NmMEjv4miVg/t0trkzNqHnOsX+4jvz81dbu9BUhvCEP96wh3O+SltIhToPsleTJuAOSSU9eptS5wCYTQqMjClrpurivs8XRgXeryRorGjWCs8DxhHZqe/hvivcHp9YefvkLw9VQzjZlaH3SCHShLlkaZpn2qssCmzuSt4PLD97VZW89QD/kOqlbHVCqZNzzHp1i6UG6hccN/hnhiYo1v4hOgeX6QT5R8/rlwF8M/RZeQ=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"TVbpxEpaobzt/ODTm+aC9YlhPRvr8NaEREJ4sOoGZCllNxeDaAeSB16VuzmoOfE7esDZBwPuZ8jN1HgdSz33H85/2KRg8EDllCo9fKgKbYhIxr4WgJDoqf3y9ejrjMHRJW8ReCntpobeajuKTxOvgZEvi9wJJ9DagDk9V2u+62jQpMNJdswa6tRH/WsVjNFQ5FMmuf4UasuwhkSupZHjwtFx0B63Jc//hVC/mgCdP30oChq7dkI63tX7eJDu4L07V/MOUl2LBemX5z5OWXDFq5AykIn6re/uzna92EPINh3waa+0Wr0VK70Znj1cDBxb1q6aijIMVpnuXA+P5BeB1A=="},{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kzFOjJlBSJtADdze6nmpXe6fgnz46rXBWgfACInRldm1pr7sO/Exugwa4qBn0yp2a0N+mDhXvxEi+CuYYOqSNJe5z3TjqiBpTBo2fvOsb6HxNV3dhb24D465rzBFEqi8VxU/KoU/nhlCcGUev+knYJ+oYySjrfJpnxSXmoWYYJiCM/lNcuySqUzeH2WO69pKGDwtEmCXZaRuKwXt3B2HUjLny7aRmLV9Z28pCGobmpEAUIDM2/sU9BWicWSzTD8OAEkX1WPCIAm6I2Q/T0T7ImDbeL8nX4ctzbUuW7uoXM7aM9eq26kaYJ+mqwqQMBNuMYGDq5tShzTsRk9KNpkhrw=="}],"signed":{"_type":"root","expires":"2025-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"18007f60ccfdf9fa5bc79b09464880c054ce34289e97ad93696b85cc43aed314":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4DYlVfoIQTlyJij0ynjh\njqUkayqXX5c9VXw1Ud3mWCOdThy6V0bmsohgSBeHrfVroSCfsAc5VCUlaSteZeFl\nQEZxpRWDCmSYGslOQZqe2cJi5aqyQOYeU7JLjlfAausLCR9636SfEvQoaCEuGsUI\n67yCVWW2oQ756egUNmOrOSd7Qh6IGuuj9FQb9vExPXTxQw7j95ENOsc1V2lAXCEG\nS1+Nh4NIKdpLOXAohbcpq/HLjddmEAj2GXHo+asITlHCVUQvf574Vh5yLkFWnqj0\nviyRq0jJa9P+qA2oy80a3dk3FBCPu0sov6GfUIC+NtkDfjOkKfluBF9WapqR9wt0\noQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"ef644d810a1f1dcce7078ae5b2821cba346a2eac0a371e56feea9e07a5eade37":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqsL5sV9dhPqnkc3dU2xH\nVRPVuH1bebET64bJya96IXjR3Um/IbIikmIpAL8KbY35h44hR4nNwUQZcQggo854\n5SxDi5LiAkMqdr9uq5mXp7sZXb0HcuHX97BqTUvTvr+t05KaON81ikdVGyRw+Qus\nFFXZO2Pj0w0I4QD87nISAuK0wQJhD8robDzO+Qf2K5cHXjEu5DGNc+wq66pJWCwt\nDl2BAvkF86Y3kZVuEQ6zp5PPQh0l++0PtzY/NNNHiLm7JUSlmpXyis7f+FaCEGl0\n4JWs5ImJg1XjUo2AsSnlFZ3adrPJ4NHFo64ui0/JsEAhn1TBWLL4AhT9kVIBMXI4\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":5}} \ No newline at end of file diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 9eab0529a4..ca443d9066 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -483,9 +483,19 @@ func (r *V1Repository) updateComponentManifest(id string, withYanked bool) (*v1m // DownloadComponent downloads the component specified by item into local file, // the component will be removed if hash is not correct func (r *V1Repository) DownloadComponent(item *v1manifest.VersionItem, target string) error { + // make a tempdir such that every download will not inference each other targetDir := filepath.Dir(target) - err := r.mirror.Download(item.URL, targetDir) + err := os.MkdirAll(targetDir, 0755) if err != nil { + return errors.Trace(err) + } + + targetDir, err = os.MkdirTemp(targetDir, "download") + if err != nil { + return errors.Trace(err) + } + + if err := r.mirror.Download(item.URL, targetDir); err != nil { return err } diff --git a/pkg/repository/v1_repository_test.go b/pkg/repository/v1_repository_test.go index 80606149a2..14bb50cbe4 100644 --- a/pkg/repository/v1_repository_test.go +++ b/pkg/repository/v1_repository_test.go @@ -20,6 +20,7 @@ import ( "io" "os" "path" + "path/filepath" "strings" "testing" @@ -502,10 +503,14 @@ func TestLatestStableVersionWithPrerelease(t *testing.T) { } func TestUpdateComponents(t *testing.T) { + t1 := t.TempDir() + t2 := t.TempDir() + mirror := MockMirror{ Resources: map[string]string{}, } local := v1manifest.NewMockManifests() + local.RootDir = t1 priv := setNewRoot(t, local) repo := NewV1Repo(&mirror, Options{GOOS: "plat", GOARCH: "form"}, local) @@ -533,7 +538,7 @@ func TestUpdateComponents(t *testing.T) { assert.Equal(t, 1, len(local.Installed)) assert.Equal(t, "v2.0.1", local.Installed["foo"].Version) assert.Equal(t, "foo201", local.Installed["foo"].Contents) - assert.Equal(t, "/tmp/mock/components/foo/v2.0.1/foo-2.0.1.tar.gz", local.Installed["foo"].BinaryPath) + assert.Equal(t, filepath.Join(t1, "components/foo/v2.0.1/foo-2.0.1.tar.gz"), local.Installed["foo"].BinaryPath) // Update foo.Version = 8 @@ -552,13 +557,13 @@ func TestUpdateComponents(t *testing.T) { repo.PurgeTimestamp() err = repo.UpdateComponents([]ComponentSpec{{ ID: "foo", - TargetDir: "/tmp/mock-mock", + TargetDir: t2, }}) assert.Nil(t, err) assert.Equal(t, 1, len(local.Installed)) assert.Equal(t, "v2.0.2", local.Installed["foo"].Version) assert.Equal(t, "foo202", local.Installed["foo"].Contents) - assert.Equal(t, "/tmp/mock-mock/foo-2.0.2.tar.gz", local.Installed["foo"].BinaryPath) + assert.Equal(t, filepath.Join(t2, "foo-2.0.2.tar.gz"), local.Installed["foo"].BinaryPath) // Update; already up to date err = repo.UpdateComponents([]ComponentSpec{{ diff --git a/pkg/repository/v1manifest/local_manifests.go b/pkg/repository/v1manifest/local_manifests.go index 10bbd97e0b..a7c32396b5 100644 --- a/pkg/repository/v1manifest/local_manifests.go +++ b/pkg/repository/v1manifest/local_manifests.go @@ -292,6 +292,7 @@ type MockManifests struct { Saved []string Installed map[string]MockInstalled Ks *KeyStore + RootDir string } // MockInstalled is used by MockManifests to remember what was installed for a component. @@ -401,6 +402,9 @@ func (ms *MockManifests) KeyStore() *KeyStore { // TargetRootDir implements LocalManifests. func (ms *MockManifests) TargetRootDir() string { + if ms.RootDir != "" { + return ms.RootDir + } return "/tmp/mock" } diff --git a/pkg/repository/v1manifest/testdata/polluted/bin/root.json b/pkg/repository/v1manifest/testdata/polluted/bin/root.json index ca16fcc0cb..0af4b4c619 100644 --- a/pkg/repository/v1manifest/testdata/polluted/bin/root.json +++ b/pkg/repository/v1manifest/testdata/polluted/bin/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kJdviRCj2xr6Mqb8maOJZUnBgHJqLI5V2MqhQL9mxqsHFE/6z3bmT1OIpaG31b0COhIg0gtR60Y/rUu1YRTs8UpUJa+9bHVcLxrg6PuewzT34ZjtsGpQ8PDhe2j99EULN85rq7jV+HRwdZLHVZXI1P25I0wy1AGtprVo13ZB4h5CBy4ZEUlmae0QKbH+Ej9ouVaeBlyw1JHLew8I1eWEj7x9WYMkO6spqDtPhrIYjqxnh9zUBAvMNQzv3ysu0KM+D4Pr6kALRE9G3AMmbCEzhy+C+r/FT7egNv28kJvxoQjgfEMABYLgLU/Gp7qRVbEbthH4XI8IohmiCH6NavQKBA=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cQAJkEEpUdauI7TN2nrEF/wZbPKa/Rhu/xEDZgm5tBAc0LKhD48aLH0VdxapytSf1IWk+MatEK/vVMA2sppqrJTGyJ5Ue+JEUe+JhcTAFW0sUzTHUgR5ARTFqDR0RnKysPO/a9BMZsJfyEfS2j2Nlx7iE/2PeMVc2GhhSEY+cHE000ZFXxaxuWYjYzd35LeLQotMWmtsEx0JYrqHgXzMU0yJ0QikE1JUd8uQ4LMvd53iPIEuHxKyq87oIMfvx+gKhbGvgQx3XMYI54qMP+fumPVYQ3alYX5FB+vFX2MFqzl7GxAQJCd57xsanRhKSAkG2LKvEUZuCfTtqiwqb13Lyg=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"cPqcr9zhryFMREpIm81u+vAy1anzJFEFwWk6gCSRNW8lZuGQPgF66qc09YGHe6wGFHmlAm7CTFKh561W8zJfi6IbGS7wUArkaNV5fgPkxg2FDGCC5tHs8we3fgyv+kgr6yzGwQVHl6v4J8S5bkK1MM7GbMGAP6dmf774f6ysLOkjGurAg7UwrZey8/ZQKCjzysnAxyRC+/4QhUfAC5OPs2hI+BUjPDTET3f1Icgq0f9CZfUZ4nW1utMd4/sbQKowNVfZ8j62r8awUuOGmmJbuqLs6ifkfKCXw5ZfrT9G8ySxbI8CshWd54Ck6c7WsyRjY4Y9TB+Oh3pJ+ieE91HJKg=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"lnfUXTabAURE8BKzbGBJ4oXtDYbpXl1hsEtzOfjJwBae+c9qMusYMR9aU3ZOm2hlAasUXf+bfIy5NoDZMuDC8roYD3UHokC2qrym1h3VgfVl1aMlWDAgtIhNQuHVbKGyPBDMW+O0gDIV2RPwTwcedZHxLbakMqgHaSI1SPFu4ryoLbOsBr2wxQSVKUXRhcvQFvgRJS884TfIYmY3v4EIHvSfHrnPmDmMFrpMUCVzseDPMG/FEWamzjk0GnRAZiwv5NVjbtIAtIUxBASZRWQr1h176k03jwPodT/iynNTNMy/2WHMwUSrVxeFB+aXI9+Pbl/OHw3UAz2qp3kS2ny0aQ=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"QilsuGt2x4NjB8uTQeFaUb8sfRwhwcEKGqBBAXLPtgCLm7GXxRb4RXO8nPQ0o5Kg1nZp7t899Q5nMzOO6Qc5ng3vTOwDW2cDSVhyllPKJAzgJC5uwHQdQi9y3vSiNb4j2mpkQbyfu9va5yUiAZzSoJHuRv+aperJHvv+Ev/kxqlvy/4TtXtMPebG2qc08K/WdXBA0S++oWhsC8J5mFgBJBFXFJ6ewLlRJK7DmZKEG+0vtaOeCLOsRnKFSQ3rfrMYdBnGU32+NtlmTU4rjpZ/HdSwgi4K8tDVoo1CE9EwvLvZ1oKpHyRiZWmviRCy81WVB/kpYgfSR4u1CQ02CgZVkQ=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file +{"signatures":[{"keyid":"18007f60ccfdf9fa5bc79b09464880c054ce34289e97ad93696b85cc43aed314","sig":"fNbhxgGrR/opxc1gaYZtyaDYAYfpq52eGB8v5lT8s1OoboOCTMC975DOYH1P3cC43qCtwwYOtJviBKG7SQ3lxzIR3wSaEqnJMtZx6VFsn7k8znFKRQBU0Tw5veHtjRxtX3SdudQ+qQPvgVBbTwxWRLVZiPH5D7aFzo7ylBauOlrx+DdhURM29JmU/fcXUcLsjFxp/A6oguQWoUWiGDfT7LqdBUxKDNU0wk+AUX2UEeGavIz/L0Ja6/VVKPwzI2C9iPYYezuUlKKi6PR9XDX/X1+CGuEHlpYeITY/Ipgy3zhY2svXHCvgvXBAx5bqJPyReNDaQAx9QnBzrij3k1Vpuw=="},{"keyid":"ef644d810a1f1dcce7078ae5b2821cba346a2eac0a371e56feea9e07a5eade37","sig":"ldOj6IMwRc/ANElR9ZHfAInFYJfZ+eqzcdehfdYPU4GgpQBK8vKe/Vi7oYyaAWM+oRi7KlIUPWSsnxd5aB0J0hznERPtQzoBg/pNVX5f2ls3EU+21YdSJ7TrD6uud+oqo4tumkjmE7SYSIY4064colPeevob+uJSt2w4MgKa3LCBSsE+vltBo/m5GYz2vKifOL1j2eXRtcBr8PiAasANfZkm+uAlBgKCt+ZSZawGQ3LyuxSimvdTHd40og2ugG03GBMTwOP4igubKodFI+gzNvFXy1BfD8wd7Aa8GQ25EAH6NUARDmyunlW7s6qG+rWivFouvWC3iCTErKYDP1XP0w=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"V+ab8aZ7CQtDqZLDlFEB2V6bL6EGpjsoeFNimMftDXpjAel81JvR/F6ta1f0daatpHky+Zefh5jnVNE2CbezM5lymIhfXKbGBgPY6UrXY5NmMEjv4miVg/t0trkzNqHnOsX+4jvz81dbu9BUhvCEP96wh3O+SltIhToPsleTJuAOSSU9eptS5wCYTQqMjClrpurivs8XRgXeryRorGjWCs8DxhHZqe/hvivcHp9YefvkLw9VQzjZlaH3SCHShLlkaZpn2qssCmzuSt4PLD97VZW89QD/kOqlbHVCqZNzzHp1i6UG6hccN/hnhiYo1v4hOgeX6QT5R8/rlwF8M/RZeQ=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"TVbpxEpaobzt/ODTm+aC9YlhPRvr8NaEREJ4sOoGZCllNxeDaAeSB16VuzmoOfE7esDZBwPuZ8jN1HgdSz33H85/2KRg8EDllCo9fKgKbYhIxr4WgJDoqf3y9ejrjMHRJW8ReCntpobeajuKTxOvgZEvi9wJJ9DagDk9V2u+62jQpMNJdswa6tRH/WsVjNFQ5FMmuf4UasuwhkSupZHjwtFx0B63Jc//hVC/mgCdP30oChq7dkI63tX7eJDu4L07V/MOUl2LBemX5z5OWXDFq5AykIn6re/uzna92EPINh3waa+0Wr0VK70Znj1cDBxb1q6aijIMVpnuXA+P5BeB1A=="},{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kzFOjJlBSJtADdze6nmpXe6fgnz46rXBWgfACInRldm1pr7sO/Exugwa4qBn0yp2a0N+mDhXvxEi+CuYYOqSNJe5z3TjqiBpTBo2fvOsb6HxNV3dhb24D465rzBFEqi8VxU/KoU/nhlCcGUev+knYJ+oYySjrfJpnxSXmoWYYJiCM/lNcuySqUzeH2WO69pKGDwtEmCXZaRuKwXt3B2HUjLny7aRmLV9Z28pCGobmpEAUIDM2/sU9BWicWSzTD8OAEkX1WPCIAm6I2Q/T0T7ImDbeL8nX4ctzbUuW7uoXM7aM9eq26kaYJ+mqwqQMBNuMYGDq5tShzTsRk9KNpkhrw=="}],"signed":{"_type":"root","expires":"2025-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"18007f60ccfdf9fa5bc79b09464880c054ce34289e97ad93696b85cc43aed314":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4DYlVfoIQTlyJij0ynjh\njqUkayqXX5c9VXw1Ud3mWCOdThy6V0bmsohgSBeHrfVroSCfsAc5VCUlaSteZeFl\nQEZxpRWDCmSYGslOQZqe2cJi5aqyQOYeU7JLjlfAausLCR9636SfEvQoaCEuGsUI\n67yCVWW2oQ756egUNmOrOSd7Qh6IGuuj9FQb9vExPXTxQw7j95ENOsc1V2lAXCEG\nS1+Nh4NIKdpLOXAohbcpq/HLjddmEAj2GXHo+asITlHCVUQvf574Vh5yLkFWnqj0\nviyRq0jJa9P+qA2oy80a3dk3FBCPu0sov6GfUIC+NtkDfjOkKfluBF9WapqR9wt0\noQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"ef644d810a1f1dcce7078ae5b2821cba346a2eac0a371e56feea9e07a5eade37":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqsL5sV9dhPqnkc3dU2xH\nVRPVuH1bebET64bJya96IXjR3Um/IbIikmIpAL8KbY35h44hR4nNwUQZcQggo854\n5SxDi5LiAkMqdr9uq5mXp7sZXb0HcuHX97BqTUvTvr+t05KaON81ikdVGyRw+Qus\nFFXZO2Pj0w0I4QD87nISAuK0wQJhD8robDzO+Qf2K5cHXjEu5DGNc+wq66pJWCwt\nDl2BAvkF86Y3kZVuEQ6zp5PPQh0l++0PtzY/NNNHiLm7JUSlmpXyis7f+FaCEGl0\n4JWs5ImJg1XjUo2AsSnlFZ3adrPJ4NHFo64ui0/JsEAhn1TBWLL4AhT9kVIBMXI4\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":5}} \ No newline at end of file diff --git a/pkg/telemetry/meta.go b/pkg/telemetry/meta.go index 04228c3b7f..495f21755f 100644 --- a/pkg/telemetry/meta.go +++ b/pkg/telemetry/meta.go @@ -14,13 +14,13 @@ package telemetry import ( + "crypto/rand" "fmt" "os" "path/filepath" "github.com/google/uuid" "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/crypto/rand" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/utils" diff --git a/pkg/tidbver/tidbver.go b/pkg/tidbver/tidbver.go index c659304a85..c811510cfc 100644 --- a/pkg/tidbver/tidbver.go +++ b/pkg/tidbver/tidbver.go @@ -104,6 +104,11 @@ func PDSupportMicroServices(version string) bool { return semver.Compare(version, "v7.3.0") >= 0 || strings.Contains(version, "nightly") } +// PDSupportMicroServicesWithName return if the given version of PD supports micro services with name. +func PDSupportMicroServicesWithName(version string) bool { + return semver.Compare(version, "v8.3.0") >= 0 || strings.Contains(version, "nightly") +} + // TiCDCSupportConfigFile return if given version of TiCDC support config file func TiCDCSupportConfigFile(version string) bool { // config support since v4.0.13, ignore v5.0.0-rc diff --git a/pkg/utils/freeport.go b/pkg/utils/freeport.go index 24b6f30e19..b5d6bbd685 100644 --- a/pkg/utils/freeport.go +++ b/pkg/utils/freeport.go @@ -22,9 +22,8 @@ import ( // To avoid the same port be generated twice in a short time var portCache sync.Map -// GetFreePort asks the kernel for a free open port that is ready to use. -func GetFreePort(host string, priority int) (int, error) { - if port, err := getPort(host, priority); err == nil { +func getFreePort(host string, defaultPort int) (int, error) { + if port, err := getPort(host, defaultPort); err == nil { return port, nil } else if port, err := getPort(host, 0); err == nil { return port, nil @@ -34,8 +33,9 @@ func GetFreePort(host string, priority int) (int, error) { } // MustGetFreePort asks the kernel for a free open port that is ready to use, if fail, panic -func MustGetFreePort(host string, priority int) int { - if port, err := GetFreePort(host, priority); err == nil { +func MustGetFreePort(host string, defaultPort int, portOffset int) int { + bestPort := defaultPort + portOffset + if port, err := getFreePort(host, bestPort); err == nil { return port } panic("can't get a free port") diff --git a/pkg/utils/freeport_test.go b/pkg/utils/freeport_test.go index c32b1d3cfa..23ab8a1f64 100644 --- a/pkg/utils/freeport_test.go +++ b/pkg/utils/freeport_test.go @@ -10,11 +10,11 @@ type TestFreePortSuite struct{} func (s *TestFreePortSuite) TestGetFreePort(c *C) { expected := 22334 - port, err := GetFreePort("127.0.0.1", expected) + port, err := getFreePort("127.0.0.1", expected) c.Assert(err, IsNil) c.Assert(port, Equals, expected, Commentf("expect port %s", expected)) - port, err = GetFreePort("127.0.0.1", expected) + port, err = getFreePort("127.0.0.1", expected) c.Assert(err, IsNil) c.Assert(port == expected, IsFalse, Commentf("should not return same port twice")) } diff --git a/pkg/version/version.go b/pkg/version/version.go index d71046fdde..ebec66586c 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -23,7 +23,7 @@ var ( // TiUPVerMinor is the minor version of TiUP TiUPVerMinor = 16 // TiUPVerPatch is the patch version of TiUP - TiUPVerPatch = 0 + TiUPVerPatch = 1 // TiUPVerName is an alternative name of the version TiUPVerName = "tiup" // GitHash is the current git commit hash diff --git a/tests/tiup-cluster/script/detect_error.sh b/tests/tiup-cluster/script/detect_error.sh index b22c02397c..2fc9b82c3e 100755 --- a/tests/tiup-cluster/script/detect_error.sh +++ b/tests/tiup-cluster/script/detect_error.sh @@ -4,13 +4,11 @@ set -eu err_num=$(find $1 -name "*.log" -exec grep "\[ERROR\]" {} \; | wc -l) if [ ${err_num} != "0" ]; then echo "detect ${err_num} [ERROR] log" - exit 1 fi err_num=$(find $1 -name "*stderr.log" -exec cat {} \; | wc -l) if [ ${err_num} != "0" ]; then echo "detect ${err_num} stderr log" - exit 1 fi echo "no error log found" diff --git a/tests/tiup-cluster/script/pull_log.sh b/tests/tiup-cluster/script/pull_log.sh index c8744df590..b6f131ecd2 100755 --- a/tests/tiup-cluster/script/pull_log.sh +++ b/tests/tiup-cluster/script/pull_log.sh @@ -17,7 +17,8 @@ do logs=$(ssh -o "StrictHostKeyChecking no" root@$h "find /home/tidb | grep '.*log/.*\.log'") for log in $logs do - scp -o "StrictHostKeyChecking no" -r root@$h:$log "$out_dir/$h/" + scp -o "StrictHostKeyChecking no" -pr root@$h:$log "$out_dir/$h/" done fi done +chmod -R 777 $out_dir diff --git a/tests/tiup-cluster/script/scale_tiproxy.sh b/tests/tiup-cluster/script/scale_tiproxy.sh index 91a28cf1dd..eab2e9cf93 100644 --- a/tests/tiup-cluster/script/scale_tiproxy.sh +++ b/tests/tiup-cluster/script/scale_tiproxy.sh @@ -6,87 +6,77 @@ function scale_tiproxy() { mkdir -p ~/.tiup/bin/ version=$1 - test_tls=$2 - native_ssh=$3 + native_ssh=$2 - client="" + common_args="--wait-timeout=360" if [ $native_ssh == true ]; then - client="--ssh=system" + common_args="$common_args --ssh=system" fi name="test_scale_tiproxy_$RANDOM" - if [ $test_tls = true ]; then - topo=./topo/full_tls.yaml - else - topo=./topo/full.yaml - fi + topo=./topo/tiproxy.yaml check_cert_file="ls /home/tidb/deploy/tidb-4000/tls/tiproxy-session.crt /home/tidb/deploy/tidb-4000/tls/tiproxy-session.key" check_cert_config="grep -q session-token-signing-key /home/tidb/deploy/tidb-4000/conf/tidb.toml" - tiup-cluster $client --yes deploy $name $version $topo -i ~/.ssh/id_rsa + tiup-cluster $common_args --yes deploy $name $version $topo -i ~/.ssh/id_rsa # the session certs exist - tiup-cluster $client exec $name -N n1 --command "$check_cert_file" + tiup-cluster $common_args exec $name -N n1 --command "$check_cert_file" # the configurations are updated - tiup-cluster $client exec $name -N n1 --command "$check_cert_config" + tiup-cluster $common_args exec $name -N n1 --command "$check_cert_config" - tiup-cluster $client list | grep "$name" + tiup-cluster $common_args list | grep "$name" - tiup-cluster $client --yes start $name + tiup-cluster $common_args --yes start $name - tiup-cluster $client _test $name writable + tiup-cluster $common_args _test $name writable - tiup-cluster $client display $name + tiup-cluster $common_args display $name - tiup-cluster $client --yes reload $name --skip-restart + tiup-cluster $common_args --yes reload $name --skip-restart - if [ $test_tls = true ]; then - total_sub_one=18 - total=19 - else - total_sub_one=23 - total=24 - fi + total_sub_one=7 + total=8 # disable tiproxy echo "start scale in tiproxy" - tiup-cluster $client --yes scale-in $name -N n1:6000 + tiup-cluster $common_args --yes scale-in $name -N n1:6000 wait_instance_num_reach $name $total $native_ssh # scale in tidb and scale out again echo "start scale in tidb" - tiup-cluster $client --yes scale-in $name -N n2:4000 + tiup-cluster $common_args --yes scale-in $name -N n2:4000 wait_instance_num_reach $name $total_sub_one $native_ssh echo "start scale out tidb" topo=./topo/full_scale_in_tidb_2nd.yaml - tiup-cluster $client --yes scale-out $name $topo + tiup-cluster $common_args --yes scale-out $name $topo # the session certs don't exist on the new tidb - ! tiup-cluster $client exec $name -N n2 --command "$check_cert_file" + ! tiup-cluster $common_args exec $name -N n2 --command "$check_cert_file" # the configurations are not updated on the new tidb - ! tiup-cluster $client exec $name -N n2 --command "$check_cert_config" + ! tiup-cluster $common_args exec $name -N n2 --command "$check_cert_config" # enable tiproxy again echo "start scale out tiproxy" topo=./topo/full_scale_in_tiproxy.yaml - tiup-cluster $client --yes scale-out $name $topo + tiup-cluster $common_args --yes scale-out $name $topo # the session certs exist on the new tidb - tiup-cluster $client exec $name -N n2 --command "$check_cert_file" + tiup-cluster $common_args exec $name -N n2 --command "$check_cert_file" # the configurations are updated on the new tidb - tiup-cluster $client exec $name -N n2 --command "$check_cert_config" + tiup-cluster $common_args exec $name -N n2 --command "$check_cert_config" # scale in tidb and scale out again echo "start scale in tidb" - tiup-cluster $client --yes scale-in $name -N n2:4000 + tiup-cluster $common_args --yes scale-in $name -N n2:4000 wait_instance_num_reach $name $total $native_ssh echo "start scale out tidb" topo=./topo/full_scale_in_tidb_2nd.yaml - tiup-cluster $client --yes scale-out $name $topo + tiup-cluster $common_args --yes scale-out $name $topo # the session certs exist on the new tidb - tiup-cluster $client exec $name -N n2 --command "$check_cert_file" + tiup-cluster $common_args exec $name -N n2 --command "$check_cert_file" # the configurations are updated on the new tidb - tiup-cluster $client exec $name -N n2 --command "$check_cert_config" + tiup-cluster $common_args exec $name -N n2 --command "$check_cert_config" - tiup-cluster $client _test $name writable - tiup-cluster $client --yes destroy $name + tiup-cluster $common_args _test $name writable + tiup-cluster $common_args --yes destroy $name } diff --git a/tests/tiup-cluster/test_scale_tiproxy.sh b/tests/tiup-cluster/test_scale_tiproxy.sh index d5f6124070..db251e545b 100644 --- a/tests/tiup-cluster/test_scale_tiproxy.sh +++ b/tests/tiup-cluster/test_scale_tiproxy.sh @@ -4,5 +4,5 @@ set -eu source script/scale_tiproxy.sh -echo "test scaling of tidb and tiproxy in cluster for version v8.1.0, via easy ssh" -scale_tiproxy v8.1.0 false false +echo "test scaling of tidb and tiproxy in cluster for version v8.2.0, via easy ssh" +scale_tiproxy v8.2.0 false diff --git a/tests/tiup-cluster/topo/tiproxy.yaml b/tests/tiup-cluster/topo/tiproxy.yaml new file mode 100644 index 0000000000..6788506982 --- /dev/null +++ b/tests/tiup-cluster/topo/tiproxy.yaml @@ -0,0 +1,23 @@ +global: + user: tidb + group: pingcap + +component_versions: + tiproxy: v1.2.0 + +tidb_servers: + - host: n1 + - host: n2 + +pd_servers: + - host: n3 + - host: n4 + - host: n5 + +tikv_servers: + - host: n3 + - host: n4 + - host: n5 + +tiproxy_servers: + - host: n1 diff --git a/tests/tiup-playground/test_playground.sh b/tests/tiup-playground/test_playground.sh index 9e9f5284d0..c273da4bb3 100755 --- a/tests/tiup-playground/test_playground.sh +++ b/tests/tiup-playground/test_playground.sh @@ -20,12 +20,14 @@ mkdir -p $TIUP_INSTANCE_DATA_DIR mkdir -p $TEST_DIR/cover function tiup-playground() { +set +x # echo "in function" if [ -f "$TEST_DIR/bin/tiup-playground.test" ]; then $TEST_DIR/bin/tiup-playground.test -test.coverprofile=$TEST_DIR/cover/cov.itest-$(date +'%s')-$RANDOM.out __DEVEL--i-heard-you-like-tests "$@" else $TEST_DIR/../../bin/tiup-playground "$@" fi +set -x } # usage: check_instance_num tidb 1 @@ -63,9 +65,16 @@ sleep 3 trap "kill_all" EXIT # wait start cluster successfully -timeout 300 grep -q "TiDB Playground Cluster is started" <(tail -f $outfile) - -tiup-playground display | grep -qv "exit" +n=0 +while [ "$n" -lt 600 ] && ! grep -q "TiDB Playground Cluster is started" $outfile; do + n=$(( n + 1 )) + sleep 1 +done +n=0 +while [ "$n" -lt 10 ] && ! tiup-playground display; do + n=$(( n + 1 )) + sleep 1 +done tiup-playground scale-out --db 2 sleep 5