Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into mysql9
Browse files Browse the repository at this point in the history
  • Loading branch information
dveeden committed Dec 10, 2024
2 parents d056e63 + f7d5cb8 commit 4df5729
Show file tree
Hide file tree
Showing 71 changed files with 1,173 additions and 336 deletions.
5 changes: 4 additions & 1 deletion .github/workflows/integrate-cluster-cmd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,15 @@ jobs:
- name: Collect component log
working-directory: ${{ env.working-directory }}
if: ${{ failure() }}
# if: always()
run: |
docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
ls ${{ env.working-directory }}
- name: Detect error log
working-directory: ${{ env.working-directory }}
if: ${{ failure() }}
# if: always()
run: |
bash ./tests/tiup-cluster/script/detect_error.sh ./logs/
Expand All @@ -105,7 +107,8 @@ jobs:

- name: Output cluster debug log
working-directory: ${{ env.working-directory }}
if: always()
if: ${{ failure() }}
# if: always()
run: |
pwd
docker ps
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/integrate-cluster-scale.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,15 @@ jobs:
- name: Collect component log
working-directory: ${{ env.working-directory }}
if: ${{ failure() }}
# if: always()
run: |
docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
ls ${{ env.working-directory }}
- name: Detect error log
working-directory: ${{ env.working-directory }}
if: ${{ failure() }}
# if: always()
run: |
bash ./tests/tiup-cluster/script/detect_error.sh ./logs/
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/integrate-dm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
# with --dev the first run will fail for unknow reason, just retry it and will success now..
run: |
cd ${{ env.working-directory }}/docker
TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./up.sh --daemon --dev --compose ./docker-compose.dm.yml || TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./up.sh --daemon --dev --compose ./docker-compose.dm.yml
TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./up.sh --daemon --dev --compose ./docker-compose.dm.yml
- name: Check running containers
run: |
Expand All @@ -87,8 +87,8 @@ jobs:
- name: Collect component log
working-directory: ${{ env.working-directory }}
if: ${{ failure() }}
# if: ${{ failure() }}
if: always()
run: |
docker exec tiup-cluster-control bash -c 'mkdir -p /tiup-cluster/logs; [[ -d ~/.tiup/logs ]] && find ~/.tiup/logs -type f -name "*.log" -exec cp {} /tiup-cluster/logs \; || true'
ls ${{ env.working-directory }}
Expand All @@ -97,7 +97,7 @@ jobs:
- name: Upload component log
if: ${{ failure() }}
# if: always()
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
with:
name: dm_logs
path: ${{ env.working-directory }}/dm.logs.tar.gz
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/integrate-playground.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ jobs:
- name: Upload component log
if: ${{ failure() }}
# if: always()
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
with:
name: playground_logs
path: ${{ env.working-directory }}/playground.logs.tar.gz
Expand Down
27 changes: 27 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,32 @@
TiUP Changelog

## [1.16.1] 2024-10-31

### New Features

- Deploy DM in `tiup-playground` (#2465, @GMHDBJD, @siddontang)

### Fixes

- Fix several racings during download, do not upgrade nightly automatically in `tiup-cluster` (#2458, @xhebox)
- Fix possible panic when `tiup-playground` failed to start (#2457, @xhebox)
- Respect `component_versions` when `tiup-cluster` scales (#2451, @djshow832)
- Code clean: replace 'math/rand' with 'crypto/rand' (#2455, @bb7133)
- Fix tiup cannot update itself when tiup comonent exist (#2443, @nexustar)
- Do not check HTTP port for TiFlash 7.1.0 or above (#2440, @Lloyd-Pottiger)
- Also hide other password args (#2436, @xhebox)

### Improvements

- Avoid unnecessary primary transfer for pdms mode in `tiup-cluster` (#2414, @HuSharp)
- Add `--port-offset` for `tiup-playground` to start multiple instances without port conflicts (#2453, @breezewish)
- Start with a name for pdms mode in `tiup-cluster` and `tiup-playground` (#2438, #2446, #2447, @HuSharp)
- Remove prometheus systemd and interrupts collector (#2445, @xhebox)
- Add example of setting the tiflash-proxy topology (#2444, @JaySon-Huang)
- Improve cluster restart messaging for `tiup-cluster` (#2442, @zph)
- Print version when filling `tiup-playground --xx.binpath` (#2334, @HuSharp)
- Only display `--comments` when needed for `tiup-playground` (#2314, @dveeden)

## [1.16.0] 2024-06-27

### New Features
Expand Down
12 changes: 8 additions & 4 deletions cmd/mirror.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,9 @@ func newMirrorShowCmd() *cobra.Command {
// the `mirror set` sub command
func newMirrorSetCmd() *cobra.Command {
var (
root string
reset bool
root string
reset bool
silent bool
)
cmd := &cobra.Command{
Use: "set <mirror-addr>",
Expand Down Expand Up @@ -198,12 +199,15 @@ The root manifest in $TIUP_HOME will be replaced with the one in given repositor
log.Errorf("Failed to set mirror: %s\n", err.Error())
return err
}
fmt.Printf("Successfully set mirror to %s\n", addr)
if !silent {
fmt.Printf("Successfully set mirror to %s\n", addr)
}
return nil
},
}
cmd.Flags().StringVarP(&root, "root", "r", root, "Specify the path of `root.json`")
cmd.Flags().BoolVar(&reset, "reset", false, "Reset mirror to use the default address.")
cmd.Flags().BoolVar(&silent, "silent", false, "Skip non-warning messages.")

return cmd
}
Expand Down Expand Up @@ -955,7 +959,7 @@ func newMirrorCloneCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "clone <target-dir> [global version]",
Example: ` tiup mirror clone /path/to/local --arch amd64,arm64 --os linux,darwin # Specify the architectures and OSs
tiup mirror clone /path/to/local --os linux v6.1.0 v5.4.0 # Specify multiple versions
tiup mirror clone /path/to/local --os linux v6.1.0 v5.4.0 # Specify multiple versions
tiup mirror clone /path/to/local --full # Build a full local mirror
tiup mirror clone /path/to/local --tikv v4 --prefix # Specify the version via prefix
tiup mirror clone /path/to/local --tidb all --pd all # Download all version for specific component`,
Expand Down
8 changes: 8 additions & 0 deletions components/playground/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ func buildCommands(tp CommandType, opt *BootOptions) (cmds []Command) {
{"ticdc", opt.TiCDC},
{"tikv-cdc", opt.TiKVCDC},
{"drainer", opt.Drainer},
{"dm-master", opt.DMMaster},
{"dm-worker", opt.DMWorker},
}

for _, cmd := range commands {
Expand Down Expand Up @@ -113,6 +115,8 @@ func newScaleOut() *cobra.Command {
cmd.Flags().StringVarP(&opt.TSO.Host, "tso.host", "", opt.TSO.Host, "Playground TSO host. If not provided, TSO will still use `host` flag as its host")
cmd.Flags().StringVarP(&opt.Scheduling.Host, "scheduling.host", "", opt.Scheduling.Host, "Playground Scheduling host. If not provided, Scheduling will still use `host` flag as its host")
cmd.Flags().StringVarP(&opt.TiProxy.Host, "tiproxy.host", "", opt.PD.Host, "Playground TiProxy host. If not provided, TiProxy will still use `host` flag as its host")
cmd.Flags().IntVarP(&opt.DMMaster.Num, "dm-master", "", opt.DMMaster.Num, "DM-master instance number")
cmd.Flags().IntVarP(&opt.DMWorker.Num, "dm-worker", "", opt.DMWorker.Num, "DM-worker instance number")

cmd.Flags().StringVarP(&opt.TiDB.ConfigPath, "db.config", "", opt.TiDB.ConfigPath, "TiDB instance configuration file")
cmd.Flags().StringVarP(&opt.TiKV.ConfigPath, "kv.config", "", opt.TiKV.ConfigPath, "TiKV instance configuration file")
Expand All @@ -123,6 +127,8 @@ func newScaleOut() *cobra.Command {
cmd.Flags().StringVarP(&opt.TiProxy.ConfigPath, "tiproxy.config", "", opt.TiProxy.ConfigPath, "TiProxy instance configuration file")
cmd.Flags().StringVarP(&opt.Pump.ConfigPath, "pump.config", "", opt.Pump.ConfigPath, "Pump instance configuration file")
cmd.Flags().StringVarP(&opt.Drainer.ConfigPath, "drainer.config", "", opt.Drainer.ConfigPath, "Drainer instance configuration file")
cmd.Flags().StringVarP(&opt.DMMaster.ConfigPath, "dm-master.config", "", opt.DMMaster.ConfigPath, "DM-master instance configuration file")
cmd.Flags().StringVarP(&opt.DMWorker.ConfigPath, "dm-worker.config", "", opt.DMWorker.ConfigPath, "DM-worker instance configuration file")

cmd.Flags().StringVarP(&opt.TiDB.BinPath, "db.binpath", "", opt.TiDB.BinPath, "TiDB instance binary path")
cmd.Flags().StringVarP(&opt.TiKV.BinPath, "kv.binpath", "", opt.TiKV.BinPath, "TiKV instance binary path")
Expand All @@ -135,6 +141,8 @@ func newScaleOut() *cobra.Command {
cmd.Flags().StringVarP(&opt.TiKVCDC.BinPath, "kvcdc.binpath", "", opt.TiKVCDC.BinPath, "TiKVCDC instance binary path")
cmd.Flags().StringVarP(&opt.Pump.BinPath, "pump.binpath", "", opt.Pump.BinPath, "Pump instance binary path")
cmd.Flags().StringVarP(&opt.Drainer.BinPath, "drainer.binpath", "", opt.Drainer.BinPath, "Drainer instance binary path")
cmd.Flags().StringVarP(&opt.DMMaster.BinPath, "dm-master.binpath", "", opt.DMMaster.BinPath, "DM-master instance binary path")
cmd.Flags().StringVarP(&opt.DMWorker.BinPath, "dm-worker.binpath", "", opt.DMWorker.BinPath, "DM-worker instance binary path")

return cmd
}
Expand Down
8 changes: 2 additions & 6 deletions components/playground/grafana.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,12 +151,8 @@ var clusterName = "Test-Cluster"

// dir should contains files untar the grafana.
// return not error iff the Cmd is started successfully.
func (g *grafana) start(ctx context.Context, dir string, p8sURL string) (err error) {
g.port, err = utils.GetFreePort(g.host, g.port)
if err != nil {
return err
}

func (g *grafana) start(ctx context.Context, dir string, portOffset int, p8sURL string) (err error) {
g.port = utils.MustGetFreePort(g.host, g.port, portOffset)
fname := filepath.Join(dir, "conf", "provisioning", "dashboards", "dashboard.yml")
err = writeDashboardConfig(fname, clusterName, filepath.Join(dir, "dashboards"))
if err != nil {
Expand Down
90 changes: 90 additions & 0 deletions components/playground/instance/dm_master.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package instance

import (
"context"
"fmt"
"path/filepath"
"strings"

"github.com/pingcap/tiup/pkg/utils"
)

// DMMaster represent a DM master instance.
type DMMaster struct {
instance
Process
initEndpoints []*DMMaster
}

var _ Instance = &DMMaster{}

// NewDMMaster create a new DMMaster instance.
func NewDMMaster(binPath string, dir, host, configPath string, portOffset int, id int, port int) *DMMaster {
if port <= 0 {
port = 8261
}
return &DMMaster{
instance: instance{
BinPath: binPath,
ID: id,
Dir: dir,
Host: host,
Port: utils.MustGetFreePort(host, 8291, portOffset),
// Similar like PD's client port, here use StatusPort for Master Port.
StatusPort: utils.MustGetFreePort(host, port, portOffset),
ConfigPath: configPath,
},
}
}

// Name return the name of the instance.
func (m *DMMaster) Name() string {
return fmt.Sprintf("dm-master-%d", m.ID)
}

// Start starts the instance.
func (m *DMMaster) Start(ctx context.Context) error {
args := []string{
fmt.Sprintf("--name=%s", m.Name()),
fmt.Sprintf("--master-addr=http://%s", utils.JoinHostPort(m.Host, m.StatusPort)),
fmt.Sprintf("--advertise-addr=http://%s", utils.JoinHostPort(AdvertiseHost(m.Host), m.StatusPort)),
fmt.Sprintf("--peer-urls=http://%s", utils.JoinHostPort(m.Host, m.Port)),
fmt.Sprintf("--advertise-peer-urls=http://%s", utils.JoinHostPort(AdvertiseHost(m.Host), m.Port)),
fmt.Sprintf("--log-file=%s", m.LogFile()),
}

endpoints := make([]string, 0)
for _, master := range m.initEndpoints {
endpoints = append(endpoints, fmt.Sprintf("%s=http://%s", master.Name(), utils.JoinHostPort(master.Host, master.Port)))
}
args = append(args, fmt.Sprintf("--initial-cluster=%s", strings.Join(endpoints, ",")))

if m.ConfigPath != "" {
args = append(args, fmt.Sprintf("--config=%s", m.ConfigPath))
}

m.Process = &process{cmd: PrepareCommand(ctx, m.BinPath, args, nil, m.Dir)}

logIfErr(m.Process.SetOutputFile(m.LogFile()))
return m.Process.Start()
}

// SetInitEndpoints set the initial endpoints for the DM master.
func (m *DMMaster) SetInitEndpoints(endpoints []*DMMaster) {
m.initEndpoints = endpoints
}

// Component return the component of the instance.
func (m *DMMaster) Component() string {
return "dm-master"
}

// LogFile return the log file path of the instance.
func (m *DMMaster) LogFile() string {
return filepath.Join(m.Dir, "dm-master.log")
}

// Addr return the address of the instance.
func (m *DMMaster) Addr() string {
return utils.JoinHostPort(m.Host, m.StatusPort)
}
83 changes: 83 additions & 0 deletions components/playground/instance/dm_worker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package instance

import (
"context"
"fmt"
"path/filepath"
"strings"

"github.com/pingcap/tiup/pkg/utils"
)

// DMWorker represent a DM worker instance.
type DMWorker struct {
instance
Process

masters []*DMMaster
}

var _ Instance = &DMWorker{}

// NewDMWorker create a DMWorker instance.
func NewDMWorker(binPath string, dir, host, configPath string, portOffset int, id int, port int, masters []*DMMaster) *DMWorker {
if port <= 0 {
port = 8262
}
return &DMWorker{
instance: instance{
BinPath: binPath,
ID: id,
Dir: dir,
Host: host,
Port: utils.MustGetFreePort(host, port, portOffset),
ConfigPath: configPath,
},
masters: masters,
}
}

// MasterAddrs return the master addresses.
func (w *DMWorker) MasterAddrs() []string {
var addrs []string
for _, master := range w.masters {
addrs = append(addrs, utils.JoinHostPort(AdvertiseHost(master.Host), master.StatusPort))
}
return addrs
}

// Name return the name of the instance.
func (w *DMWorker) Name() string {
return fmt.Sprintf("dm-worker-%d", w.ID)
}

// Start starts the instance.
func (w *DMWorker) Start(ctx context.Context) error {
args := []string{
fmt.Sprintf("--name=%s", w.Name()),
fmt.Sprintf("--worker-addr=%s", utils.JoinHostPort(w.Host, w.Port)),
fmt.Sprintf("--advertise-addr=%s", utils.JoinHostPort(AdvertiseHost(w.Host), w.Port)),
fmt.Sprintf("--join=%s", strings.Join(w.MasterAddrs(), ",")),
fmt.Sprintf("--log-file=%s", w.LogFile()),
}

if w.ConfigPath != "" {
args = append(args, fmt.Sprintf("--config=%s", w.ConfigPath))
}

w.Process = &process{cmd: PrepareCommand(ctx, w.BinPath, args, nil, w.Dir)}

logIfErr(w.Process.SetOutputFile(w.LogFile()))

return w.Process.Start()
}

// Component return the component of the instance.
func (w *DMWorker) Component() string {
return "dm-worker"
}

// LogFile return the log file of the instance.
func (w *DMWorker) LogFile() string {
return filepath.Join(w.Dir, "dm-worker.log")
}
4 changes: 2 additions & 2 deletions components/playground/instance/drainer.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@ type Drainer struct {
var _ Instance = &Drainer{}

// NewDrainer create a Drainer instance.
func NewDrainer(binPath string, dir, host, configPath string, id int, pds []*PDInstance) *Drainer {
func NewDrainer(binPath string, dir, host, configPath string, portOffset int, id int, pds []*PDInstance) *Drainer {
d := &Drainer{
instance: instance{
BinPath: binPath,
ID: id,
Dir: dir,
Host: host,
Port: utils.MustGetFreePort(host, 8250),
Port: utils.MustGetFreePort(host, 8250, portOffset),
ConfigPath: configPath,
},
pds: pds,
Expand Down
Loading

0 comments on commit 4df5729

Please sign in to comment.