diff --git a/components/playground/playground.go b/components/playground/playground.go index 486b3495a4..4b478a0a56 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -176,7 +176,7 @@ func (p *Playground) removePumpWhenTombstone(c *api.BinlogClient, inst *instance defer logIfErr(p.renderSDFile()) for { - tombstone, err := c.IsPumpTombstone(inst.NodeID()) + tombstone, err := c.IsPumpTombstone(inst.Addr()) if err != nil { fmt.Println(err) } @@ -199,7 +199,7 @@ func (p *Playground) removeDrainerWhenTombstone(c *api.BinlogClient, inst *insta defer logIfErr(p.renderSDFile()) for { - tombstone, err := c.IsDrainerTombstone(inst.NodeID()) + tombstone, err := c.IsDrainerTombstone(inst.Addr()) if err != nil { fmt.Println(err) } @@ -325,7 +325,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { if err != nil { return errors.AddStack(err) } - err = c.OfflinePump(inst.Addr(), inst.NodeID()) + err = c.OfflinePump(inst.Addr()) if err != nil { return errors.AddStack(err) } @@ -344,7 +344,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { if err != nil { return errors.AddStack(err) } - err = c.OfflineDrainer(inst.Addr(), inst.NodeID()) + err = c.OfflineDrainer(inst.Addr()) if err != nil { return errors.AddStack(err) } diff --git a/pkg/cluster/api/binlog.go b/pkg/cluster/api/binlog.go index 83fffa5a6e..ee57066097 100644 --- a/pkg/cluster/api/binlog.go +++ b/pkg/cluster/api/binlog.go @@ -84,12 +84,20 @@ type NodeStatus struct { } // IsPumpTombstone check if drainer is tombstone. -func (c *BinlogClient) IsPumpTombstone(nodeID string) (bool, error) { +func (c *BinlogClient) IsPumpTombstone(addr string) (bool, error) { + nodeID, err := c.nodeID(addr, "pumps") + if err != nil { + return false, err + } return c.isTombstone("pumps", nodeID) } // IsDrainerTombstone check if drainer is tombstone. -func (c *BinlogClient) IsDrainerTombstone(nodeID string) (bool, error) { +func (c *BinlogClient) IsDrainerTombstone(addr string) (bool, error) { + nodeID, err := c.nodeID(addr, "drainers") + if err != nil { + return false, err + } return c.isTombstone("drainer", nodeID) } @@ -121,13 +129,36 @@ func (c *BinlogClient) drainerNodeStatus() (status []*NodeStatus, err error) { return c.nodeStatus("drainers") } +func (c *BinlogClient) nodeID(addr, ty string) (string, error) { + nodes, err := c.nodeStatus(ty) + if err != nil { + return "", err + } + + for _, node := range nodes { + if addr == node.Addr { + return node.NodeID, nil + } + } + + return "", errors.Errorf("pump node id for address %s not found", addr) +} + // UpdateDrainerState update the specify state as the specified state. -func (c *BinlogClient) UpdateDrainerState(nodeID string, state string) error { +func (c *BinlogClient) UpdateDrainerState(addr string, state string) error { + nodeID, err := c.nodeID(addr, "drainers") + if err != nil { + return err + } return c.updateStatus("drainers", nodeID, state) } // UpdatePumpState update the specify state as the specified state. -func (c *BinlogClient) UpdatePumpState(nodeID string, state string) error { +func (c *BinlogClient) UpdatePumpState(addr string, state string) error { + nodeID, err := c.nodeID(addr, "pumps") + if err != nil { + return err + } return c.updateStatus("pumps", nodeID, state) } @@ -228,11 +259,19 @@ func (c *BinlogClient) offline(addr string, nodeID string) error { } // OfflinePump offline a pump. -func (c *BinlogClient) OfflinePump(addr string, nodeID string) error { +func (c *BinlogClient) OfflinePump(addr string) error { + nodeID, err := c.nodeID(addr, "pumps") + if err != nil { + return err + } return c.offline(addr, nodeID) } // OfflineDrainer offline a drainer. -func (c *BinlogClient) OfflineDrainer(addr string, nodeID string) error { +func (c *BinlogClient) OfflineDrainer(addr string) error { + nodeID, err := c.nodeID(addr, "drainers") + if err != nil { + return err + } return c.offline(addr, nodeID) } diff --git a/pkg/cluster/embed/autogen_pkger.go b/pkg/cluster/embed/autogen_pkger.go index 60af7d427a..c5cc56aacb 100644 --- a/pkg/cluster/embed/autogen_pkger.go +++ b/pkg/cluster/embed/autogen_pkger.go @@ -31,13 +31,13 @@ func init() { autogenFiles["/templates/scripts/run_dm-master.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KY2QgIiR7REVQTE9ZX0RJUn0iIHx8IGV4aXQgMQoKe3stIGRlZmluZSAiTWFzdGVyTGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkbWFzdGVyIDo9IC59fQogICAge3stIGlmIGVxICRpZHggMH19CiAgICAgIHt7LSAkbWFzdGVyLk5hbWV9fT17eyRtYXN0ZXIuU2NoZW1lfX06Ly97eyRtYXN0ZXIuSVB9fTp7eyRtYXN0ZXIuUGVlclBvcnR9fQogICAge3stIGVsc2UgLX19CiAgICAgICx7ey0gJG1hc3Rlci5OYW1lfX09e3skbWFzdGVyLlNjaGVtZX19Oi8ve3skbWFzdGVyLklQfX06e3skbWFzdGVyLlBlZXJQb3J0fX0KICAgIHt7LSBlbmR9fQogIHt7LSBlbmR9fQp7ey0gZW5kfX0KCnt7LSBpZiAuTnVtYU5vZGV9fQpleGVjIG51bWFjdGwgLS1jcHVub2RlYmluZD17ey5OdW1hTm9kZX19IC0tbWVtYmluZD17ey5OdW1hTm9kZX19IGJpbi9kbS1tYXN0ZXIvZG0tbWFzdGVyIFwKe3stIGVsc2V9fQpleGVjIGJpbi9kbS1tYXN0ZXIvZG0tbWFzdGVyIFwKe3stIGVuZH19Cnt7LSBpZiAuVjFTb3VyY2VQYXRofX0KICAgIC0tdjEtc291cmNlcy1wYXRoPSJ7ey5WMVNvdXJjZVBhdGh9fSIgXAp7ey0gZW5kfX0KICAgIC0tbmFtZT0ie3suTmFtZX19IiBcCiAgICAtLW1hc3Rlci1hZGRyPSIwLjAuMC4wOnt7LlBvcnR9fSIgXAogICAgLS1hZHZlcnRpc2UtYWRkcj0ie3suSVB9fTp7ey5Qb3J0fX0iIFwKICAgIC0tcGVlci11cmxzPSJ7ey5JUH19Ont7LlBlZXJQb3J0fX0iIFwKICAgIC0tYWR2ZXJ0aXNlLXBlZXItdXJscz0ie3suSVB9fTp7ey5QZWVyUG9ydH19IiBcCiAgICAtLWxvZy1maWxlPSJ7ey5Mb2dEaXJ9fS9kbS1tYXN0ZXIubG9nIiBcCiAgICAtLWRhdGEtZGlyPSJ7ey5EYXRhRGlyfX0iIFwKICAgIC0taW5pdGlhbC1jbHVzdGVyPSJ7e3RlbXBsYXRlICJNYXN0ZXJMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tY29uZmlnPWNvbmYvZG0tbWFzdGVyLnRvbWwgPj4gInt7LkxvZ0Rpcn19L2RtLW1hc3Rlcl9zdGRvdXQubG9nIiAyPj4gInt7LkxvZ0Rpcn19L2RtLW1hc3Rlcl9zdGRlcnIubG9nIgo=" autogenFiles["/templates/scripts/run_dm-master_scale.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KY2QgIiR7REVQTE9ZX0RJUn0iIHx8IGV4aXQgMQoKe3stIGRlZmluZSAiTWFzdGVyTGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkbWFzdGVyIDo9IC59fQogICAge3stIGlmIGVxICRpZHggMH19CiAgICAgIHt7LSAkbWFzdGVyLklQfX06e3skbWFzdGVyLlBvcnR9fQogICAge3stIGVsc2UgLX19CiAgICAgICx7ey0gJG1hc3Rlci5JUH19Ont7JG1hc3Rlci5Qb3J0fX0KICAgIHt7LSBlbmR9fQogIHt7LSBlbmR9fQp7ey0gZW5kfX0KCnt7LSBpZiAuTnVtYU5vZGV9fQpleGVjIG51bWFjdGwgLS1jcHVub2RlYmluZD17ey5OdW1hTm9kZX19IC0tbWVtYmluZD17ey5OdW1hTm9kZX19IGJpbi9kbS1tYXN0ZXIvZG0tbWFzdGVyIFwKe3stIGVsc2V9fQpleGVjIGJpbi9kbS1tYXN0ZXIvZG0tbWFzdGVyIFwKe3stIGVuZH19CiAgICAtLW5hbWU9Int7Lk5hbWV9fSIgXAogICAgLS1tYXN0ZXItYWRkcj0iMC4wLjAuMDp7ey5Qb3J0fX0iIFwKICAgIC0tYWR2ZXJ0aXNlLWFkZHI9Int7LklQfX06e3suUG9ydH19IiBcCiAgICAtLXBlZXItdXJscz0ie3suU2NoZW1lfX06Ly97ey5JUH19Ont7LlBlZXJQb3J0fX0iIFwKICAgIC0tYWR2ZXJ0aXNlLXBlZXItdXJscz0ie3suU2NoZW1lfX06Ly97ey5JUH19Ont7LlBlZXJQb3J0fX0iIFwKICAgIC0tbG9nLWZpbGU9Int7LkxvZ0Rpcn19L2RtLW1hc3Rlci5sb2ciIFwKICAgIC0tZGF0YS1kaXI9Int7LkRhdGFEaXJ9fSIgXAogICAgLS1qb2luPSJ7e3RlbXBsYXRlICJNYXN0ZXJMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tY29uZmlnPWNvbmYvZG0tbWFzdGVyLnRvbWwgPj4gInt7LkxvZ0Rpcn19L2RtLW1hc3Rlcl9zdGRvdXQubG9nIiAyPj4gInt7LkxvZ0Rpcn19L2RtLW1hc3Rlcl9zdGRlcnIubG9nIgo=" autogenFiles["/templates/scripts/run_dm-worker.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIk1hc3Rlckxpc3QifX0KICB7ey0gcmFuZ2UgJGlkeCwgJG1hc3RlciA6PSAufX0KICAgIHt7LSBpZiBlcSAkaWR4IDB9fQogICAgICB7ey0gJG1hc3Rlci5JUH19Ont7JG1hc3Rlci5Qb3J0fX0KICAgIHt7LSBlbHNlIC19fQogICAgICAse3skbWFzdGVyLklQfX06e3skbWFzdGVyLlBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gYmluL2RtLXdvcmtlci9kbS13b3JrZXIgXAp7ey0gZWxzZX19CmV4ZWMgYmluL2RtLXdvcmtlci9kbS13b3JrZXIgXAp7ey0gZW5kfX0KICAgIC0tbmFtZT0ie3suTmFtZX19IiBcCiAgICAtLXdvcmtlci1hZGRyPSIwLjAuMC4wOnt7LlBvcnR9fSIgXAogICAgLS1hZHZlcnRpc2UtYWRkcj0ie3suSVB9fTp7ey5Qb3J0fX0iIFwKICAgIC0tbG9nLWZpbGU9Int7LkxvZ0Rpcn19L2RtLXdvcmtlci5sb2ciIFwKICAgIC0tam9pbj0ie3t0ZW1wbGF0ZSAiTWFzdGVyTGlzdCIgLkVuZHBvaW50c319IiBcCiAgICAtLWNvbmZpZz1jb25mL2RtLXdvcmtlci50b21sID4+ICJ7ey5Mb2dEaXJ9fS9kbS13b3JrZXJfc3Rkb3V0LmxvZyIgMj4+ICJ7ey5Mb2dEaXJ9fS9kbS13b3JrZXJfc3RkZXJyLmxvZyIK" - autogenFiles["/templates/scripts/run_drainer.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5TY2hlbWV9fTovL3t7JHBkLklQfX06e3skcGQuQ2xpZW50UG9ydH19CiAgICB7ey0gZWxzZSAtfX0KICAgICAgLHt7LSAkcGQuU2NoZW1lfX06Ly97eyRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gYmluL2RyYWluZXIgXAp7ey0gZWxzZX19CmV4ZWMgYmluL2RyYWluZXIgXAp7ey0gZW5kfX0KICAgIC0tbm9kZS1pZD0ie3suTm9kZUlEfX0iIFwKICAgIC0tYWRkcj0ie3suSVB9fTp7ey5Qb3J0fX0iIFwKICAgIC0tcGQtdXJscz0ie3t0ZW1wbGF0ZSAiUERMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tZGF0YS1kaXI9Int7LkRhdGFEaXJ9fSIgXAogICAgLS1sb2ctZmlsZT0ie3suTG9nRGlyfX0vZHJhaW5lci5sb2ciIFwKICAgIC0tY29uZmlnPWNvbmYvZHJhaW5lci50b21sIFwKICAgIC0taW5pdGlhbC1jb21taXQtdHM9Int7LkNvbW1pdFRzfX0iIDI+PiAie3suTG9nRGlyfX0vZHJhaW5lcl9zdGRlcnIubG9nIgo=" + autogenFiles["/templates/scripts/run_drainer.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5TY2hlbWV9fTovL3t7JHBkLklQfX06e3skcGQuQ2xpZW50UG9ydH19CiAgICB7ey0gZWxzZSAtfX0KICAgICAgLHt7LSAkcGQuU2NoZW1lfX06Ly97eyRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gYmluL2RyYWluZXIgXAp7ey0gZWxzZX19CmV4ZWMgYmluL2RyYWluZXIgXAp7ey0gZW5kfX0Ke3stIGlmIC5Ob2RlSUR9fQogICAgLS1ub2RlLWlkPSJ7ey5Ob2RlSUR9fSIgXAp7ey0gZW5kfX0KICAgIC0tYWRkcj0ie3suSVB9fTp7ey5Qb3J0fX0iIFwKICAgIC0tcGQtdXJscz0ie3t0ZW1wbGF0ZSAiUERMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tZGF0YS1kaXI9Int7LkRhdGFEaXJ9fSIgXAogICAgLS1sb2ctZmlsZT0ie3suTG9nRGlyfX0vZHJhaW5lci5sb2ciIFwKICAgIC0tY29uZmlnPWNvbmYvZHJhaW5lci50b21sIFwKICAgIC0taW5pdGlhbC1jb21taXQtdHM9Int7LkNvbW1pdFRzfX0iIDI+PiAie3suTG9nRGlyfX0vZHJhaW5lcl9zdGRlcnIubG9nIgo=" autogenFiles["/templates/scripts/run_grafana.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KY2QgIiR7REVQTE9ZX0RJUn0iIHx8IGV4aXQgMQoKTEFORz1lbl9VUy5VVEYtOCBcCnt7LSBpZiAuTnVtYU5vZGV9fQpleGVjIG51bWFjdGwgLS1jcHVub2RlYmluZD17ey5OdW1hTm9kZX19IC0tbWVtYmluZD17ey5OdW1hTm9kZX19IGJpbi9iaW4vZ3JhZmFuYS1zZXJ2ZXIgXAp7ey0gZWxzZX19CmV4ZWMgYmluL2Jpbi9ncmFmYW5hLXNlcnZlciBcCnt7LSBlbmR9fQogICAgLS1ob21lcGF0aD0ie3suRGVwbG95RGlyfX0vYmluIiBcCiAgICAtLWNvbmZpZz0ie3suRGVwbG95RGlyfX0vY29uZi9ncmFmYW5hLmluaSIK" autogenFiles["/templates/scripts/run_node_exporter.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KY2QgIiR7REVQTE9ZX0RJUn0iIHx8IGV4aXQgMQoKZXhlYyA+ID4odGVlIC1pIC1hICJ7ey5Mb2dEaXJ9fS9ub2RlX2V4cG9ydGVyLmxvZyIpCmV4ZWMgMj4mMQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gYmluL25vZGVfZXhwb3J0ZXIvbm9kZV9leHBvcnRlciBcCnt7LSBlbHNlfX0KZXhlYyBiaW4vbm9kZV9leHBvcnRlci9ub2RlX2V4cG9ydGVyIFwKe3stIGVuZH19CiAgICAtLXdlYi5saXN0ZW4tYWRkcmVzcz0iOnt7LlBvcnR9fSIgXAogICAgLS1jb2xsZWN0b3IudGNwc3RhdCBcCiAgICAtLWNvbGxlY3Rvci5zeXN0ZW1kIFwKICAgIC0tY29sbGVjdG9yLm1vdW50c3RhdHMgXAogICAgLS1jb2xsZWN0b3IubWVtaW5mb19udW1hIFwKICAgIC0tY29sbGVjdG9yLmludGVycnVwdHMgXAogICAgLS1jb2xsZWN0b3IuYnVkZHlpbmZvIFwKICAgIC0tY29sbGVjdG9yLnZtc3RhdC5maWVsZHM9Il4uKiIgXAogICAgLS1sb2cubGV2ZWw9ImluZm8iCg==" autogenFiles["/templates/scripts/run_pd.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5OYW1lfX09e3skcGQuU2NoZW1lfX06Ly97eyRwZC5JUH19Ont7JHBkLlBlZXJQb3J0fX0KICAgIHt7LSBlbHNlIC19fQogICAgICAse3stICRwZC5OYW1lfX09e3skcGQuU2NoZW1lfX06Ly97eyRwZC5JUH19Ont7JHBkLlBlZXJQb3J0fX0KICAgIHt7LSBlbmR9fQogIHt7LSBlbmR9fQp7ey0gZW5kfX0KCnt7LSBpZiAuTnVtYU5vZGV9fQpleGVjIG51bWFjdGwgLS1jcHVub2RlYmluZD17ey5OdW1hTm9kZX19IC0tbWVtYmluZD17ey5OdW1hTm9kZX19IGJpbi9wZC1zZXJ2ZXIgXAp7ey0gZWxzZX19CmV4ZWMgYmluL3BkLXNlcnZlciBcCnt7LSBlbmR9fQogICAgLS1uYW1lPSJ7ey5OYW1lfX0iIFwKICAgIC0tY2xpZW50LXVybHM9Int7LlNjaGVtZX19Oi8ve3suTGlzdGVuSG9zdH19Ont7LkNsaWVudFBvcnR9fSIgXAogICAgLS1hZHZlcnRpc2UtY2xpZW50LXVybHM9Int7LlNjaGVtZX19Oi8ve3suSVB9fTp7ey5DbGllbnRQb3J0fX0iIFwKICAgIC0tcGVlci11cmxzPSJ7ey5TY2hlbWV9fTovL3t7LklQfX06e3suUGVlclBvcnR9fSIgXAogICAgLS1hZHZlcnRpc2UtcGVlci11cmxzPSJ7ey5TY2hlbWV9fTovL3t7LklQfX06e3suUGVlclBvcnR9fSIgXAogICAgLS1kYXRhLWRpcj0ie3suRGF0YURpcn19IiBcCiAgICAtLWluaXRpYWwtY2x1c3Rlcj0ie3t0ZW1wbGF0ZSAiUERMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tY29uZmlnPWNvbmYvcGQudG9tbCBcCiAgICAtLWxvZy1maWxlPSJ7ey5Mb2dEaXJ9fS9wZC5sb2ciIDI+PiAie3suTG9nRGlyfX0vcGRfc3RkZXJyLmxvZyIKICAK" autogenFiles["/templates/scripts/run_pd_scale.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5TY2hlbWV9fTovL3t7JHBkLklQfX06e3skcGQuQ2xpZW50UG9ydH19CiAgICB7ey0gZWxzZSAtfX0KICAgICAgLHt7LSAkcGQuU2NoZW1lfX06Ly97eyRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gYmluL3BkLXNlcnZlciBcCnt7LSBlbHNlfX0KZXhlYyBiaW4vcGQtc2VydmVyIFwKe3stIGVuZH19CiAgICAtLW5hbWU9Int7Lk5hbWV9fSIgXAogICAgLS1jbGllbnQtdXJscz0ie3suU2NoZW1lfX06Ly97ey5MaXN0ZW5Ib3N0fX06e3suQ2xpZW50UG9ydH19IiBcCiAgICAtLWFkdmVydGlzZS1jbGllbnQtdXJscz0ie3suU2NoZW1lfX06Ly97ey5JUH19Ont7LkNsaWVudFBvcnR9fSIgXAogICAgLS1wZWVyLXVybHM9Int7LlNjaGVtZX19Oi8ve3suSVB9fTp7ey5QZWVyUG9ydH19IiBcCiAgICAtLWFkdmVydGlzZS1wZWVyLXVybHM9Int7LlNjaGVtZX19Oi8ve3suSVB9fTp7ey5QZWVyUG9ydH19IiBcCiAgICAtLWRhdGEtZGlyPSJ7ey5EYXRhRGlyfX0iIFwKICAgIC0tam9pbj0ie3t0ZW1wbGF0ZSAiUERMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tY29uZmlnPWNvbmYvcGQudG9tbCBcCiAgICAtLWxvZy1maWxlPSJ7ey5Mb2dEaXJ9fS9wZC5sb2ciIDI+PiAie3suTG9nRGlyfX0vcGRfc3RkZXJyLmxvZyIKICAK" autogenFiles["/templates/scripts/run_prometheus.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgpERVBMT1lfRElSPXt7LkRlcGxveURpcn19CmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCiMgV0FSTklORzogVGhpcyBmaWxlIHdhcyBhdXRvLWdlbmVyYXRlZC4gRG8gbm90IGVkaXQhCiMgICAgICAgICAgQWxsIHlvdXIgZWRpdCBtaWdodCBiZSBvdmVyd3JpdHRlbiEKCmV4ZWMgPiA+KHRlZSAtaSAtYSAie3suTG9nRGlyfX0vcHJvbWV0aGV1cy5sb2ciKQpleGVjIDI+JjEKCnt7LSBpZiAuTnVtYU5vZGV9fQpleGVjIG51bWFjdGwgLS1jcHVub2RlYmluZD17ey5OdW1hTm9kZX19IC0tbWVtYmluZD17ey5OdW1hTm9kZX19IGJpbi9wcm9tZXRoZXVzL3Byb21ldGhldXMgXAp7ey0gZWxzZX19CmV4ZWMgYmluL3Byb21ldGhldXMvcHJvbWV0aGV1cyBcCnt7LSBlbmR9fQogICAgLS1jb25maWcuZmlsZT0ie3suRGVwbG95RGlyfX0vY29uZi9wcm9tZXRoZXVzLnltbCIgXAogICAgLS13ZWIubGlzdGVuLWFkZHJlc3M9Ijp7ey5Qb3J0fX0iIFwKICAgIC0td2ViLmV4dGVybmFsLXVybD0iaHR0cDovL3t7LklQfX06e3suUG9ydH19LyIgXAogICAgLS13ZWIuZW5hYmxlLWFkbWluLWFwaSBcCiAgICAtLWxvZy5sZXZlbD0iaW5mbyIgXAogICAgLS1zdG9yYWdlLnRzZGIucGF0aD0ie3suRGF0YURpcn19IiBcCiAgICAtLXN0b3JhZ2UudHNkYi5yZXRlbnRpb249Int7LlJldGVudGlvbn19Igo=" - autogenFiles["/templates/scripts/run_pump.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5TY2hlbWV9fTovL3t7JHBkLklQfX06e3skcGQuQ2xpZW50UG9ydH19CiAgICB7ey0gZWxzZSAtfX0KICAgICAgLHt7LSAkcGQuU2NoZW1lfX06Ly97eyRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gYmluL3B1bXAgXAp7ey0gZWxzZX19CmV4ZWMgYmluL3B1bXAgXAp7ey0gZW5kfX0KICAgIC0tbm9kZS1pZD0ie3suTm9kZUlEfX0iIFwKICAgIC0tYWRkcj0iMC4wLjAuMDp7ey5Qb3J0fX0iIFwKICAgIC0tYWR2ZXJ0aXNlLWFkZHI9Int7Lkhvc3R9fTp7ey5Qb3J0fX0iIFwKICAgIC0tcGQtdXJscz0ie3t0ZW1wbGF0ZSAiUERMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tZGF0YS1kaXI9Int7LkRhdGFEaXJ9fSIgXAogICAgLS1sb2ctZmlsZT0ie3suTG9nRGlyfX0vcHVtcC5sb2ciIFwKICAgIC0tY29uZmlnPWNvbmYvcHVtcC50b21sIDI+PiAie3suTG9nRGlyfX0vcHVtcF9zdGRlcnIubG9nIgo=" + autogenFiles["/templates/scripts/run_pump.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5TY2hlbWV9fTovL3t7JHBkLklQfX06e3skcGQuQ2xpZW50UG9ydH19CiAgICB7ey0gZWxzZSAtfX0KICAgICAgLHt7LSAkcGQuU2NoZW1lfX06Ly97eyRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gYmluL3B1bXAgXAp7ey0gZWxzZX19CmV4ZWMgYmluL3B1bXAgXAp7ey0gZW5kfX0Ke3stIGlmIC5Ob2RlSUR9fQogICAgLS1ub2RlLWlkPSJ7ey5Ob2RlSUR9fSIgXAp7ey0gZW5kfX0KICAgIC0tYWRkcj0iMC4wLjAuMDp7ey5Qb3J0fX0iIFwKICAgIC0tYWR2ZXJ0aXNlLWFkZHI9Int7Lkhvc3R9fTp7ey5Qb3J0fX0iIFwKICAgIC0tcGQtdXJscz0ie3t0ZW1wbGF0ZSAiUERMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tZGF0YS1kaXI9Int7LkRhdGFEaXJ9fSIgXAogICAgLS1sb2ctZmlsZT0ie3suTG9nRGlyfX0vcHVtcC5sb2ciIFwKICAgIC0tY29uZmlnPWNvbmYvcHVtcC50b21sIDI+PiAie3suTG9nRGlyfX0vcHVtcF9zdGRlcnIubG9nIgo=" autogenFiles["/templates/scripts/run_tidb.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVsc2UgLX19CiAgICAgICx7eyRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gZW52IEdPREVCVUc9bWFkdmRvbnRuZWVkPTEgYmluL3RpZGItc2VydmVyIFwKe3stIGVsc2V9fQpleGVjIGVudiBHT0RFQlVHPW1hZHZkb250bmVlZD0xIGJpbi90aWRiLXNlcnZlciBcCnt7LSBlbmR9fQogICAgLVAge3suUG9ydH19IFwKICAgIC0tc3RhdHVzPSJ7ey5TdGF0dXNQb3J0fX0iIFwKICAgIC0taG9zdD0ie3suTGlzdGVuSG9zdH19IiBcCiAgICAtLWFkdmVydGlzZS1hZGRyZXNzPSJ7ey5JUH19IiBcCiAgICAtLXN0b3JlPSJ0aWt2IiBcCiAgICAtLXBhdGg9Int7dGVtcGxhdGUgIlBETGlzdCIgLkVuZHBvaW50c319IiBcCiAgICAtLWxvZy1zbG93LXF1ZXJ5PSJsb2cvdGlkYl9zbG93X3F1ZXJ5LmxvZyIgXAogICAgLS1jb25maWc9Y29uZi90aWRiLnRvbWwgXAogICAgLS1sb2ctZmlsZT0ie3suTG9nRGlyfX0vdGlkYi5sb2ciIDI+PiAie3suTG9nRGlyfX0vdGlkYl9zdGRlcnIubG9nIgo=" autogenFiles["/templates/scripts/run_tiflash.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCmNkICJ7ey5EZXBsb3lEaXJ9fSIgfHwgZXhpdCAxCgpleHBvcnQgUlVTVF9CQUNLVFJBQ0U9MQoKZXhwb3J0IFRaPSR7VFo6LS9ldGMvbG9jYWx0aW1lfQpleHBvcnQgTERfTElCUkFSWV9QQVRIPXt7LkRlcGxveURpcn19L2Jpbi90aWZsYXNoOiRMRF9MSUJSQVJZX1BBVEgKCmVjaG8gLW4gJ3N5bmMgLi4uICcKc3RhdD0kKHRpbWUgc3luYykKZWNobyBvawplY2hvICRzdGF0Cgp7ey0gaWYgLk51bWFOb2RlfX0KZXhlYyBudW1hY3RsIC0tY3B1bm9kZWJpbmQ9e3suTnVtYU5vZGV9fSAtLW1lbWJpbmQ9e3suTnVtYU5vZGV9fSAgXAp7ey0gZWxzZX19CmV4ZWMgXAp7ey0gZW5kfX0KICAgIGJpbi90aWZsYXNoL3RpZmxhc2ggc2VydmVyIC0tY29uZmlnLWZpbGUgY29uZi90aWZsYXNoLnRvbWw=" autogenFiles["/templates/scripts/run_tikv.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCmNkICJ7ey5EZXBsb3lEaXJ9fSIgfHwgZXhpdCAxCgplY2hvIC1uICdzeW5jIC4uLiAnCnN0YXQ9JCh0aW1lIHN5bmMgfHwgc3luYykKZWNobyBvawplY2hvICRzdGF0Cgp7ey0gZGVmaW5lICJQRExpc3QifX0KICB7ey0gcmFuZ2UgJGlkeCwgJHBkIDo9IC59fQogICAge3stIGlmIGVxICRpZHggMH19CiAgICAgIHt7LSAkcGQuSVB9fTp7eyRwZC5DbGllbnRQb3J0fX0KICAgIHt7LSBlbHNlIC19fQogICAgICAse3skcGQuSVB9fTp7eyRwZC5DbGllbnRQb3J0fX0KICAgIHt7LSBlbmR9fQogIHt7LSBlbmR9fQp7ey0gZW5kfX0KCnt7LSBpZiAuTnVtYU5vZGV9fQpleGVjIG51bWFjdGwgLS1jcHVub2RlYmluZD17ey5OdW1hTm9kZX19IC0tbWVtYmluZD17ey5OdW1hTm9kZX19IGJpbi90aWt2LXNlcnZlciBcCnt7LSBlbHNlfX0KZXhlYyBiaW4vdGlrdi1zZXJ2ZXIgXAp7ey0gZW5kfX0KICAgIC0tYWRkciAie3suTGlzdGVuSG9zdH19Ont7LlBvcnR9fSIgXAogICAgLS1hZHZlcnRpc2UtYWRkciAie3suSVB9fTp7ey5Qb3J0fX0iIFwKICAgIC0tc3RhdHVzLWFkZHIgInt7Lkxpc3Rlbkhvc3R9fTp7ey5TdGF0dXNQb3J0fX0iIFwKICAgIC0tcGQgInt7dGVtcGxhdGUgIlBETGlzdCIgLkVuZHBvaW50c319IiBcCiAgICAtLWRhdGEtZGlyICJ7ey5EYXRhRGlyfX0iIFwKICAgIC0tY29uZmlnIGNvbmYvdGlrdi50b21sIFwKICAgIC0tbG9nLWZpbGUgInt7LkxvZ0Rpcn19L3Rpa3YubG9nIiAyPj4gInt7LkxvZ0Rpcn19L3Rpa3Zfc3RkZXJyLmxvZyIK" diff --git a/pkg/cluster/manager.go b/pkg/cluster/manager.go index 9e950b579a..ee66582c0c 100644 --- a/pkg/cluster/manager.go +++ b/pkg/cluster/manager.go @@ -611,7 +611,7 @@ func (m *Manager) Display(clusterName string, opt operator.Options) error { // Check if there is some instance in tombstone state nodes, _ := operator.DestroyTombstone(ctx, t, true /* returnNodesOnly */, opt, tlsCfg) if len(nodes) != 0 { - color.Green("There are some nodes in state: `Tombstone`\n\tNodes: %+v\n\tYou can destroy them with the command: `tiup cluster prune %s`", nodes, clusterName) + color.Green("There are some nodes can be pruned: \n\tNodes: %+v\n\tYou can destroy them with the command: `tiup cluster prune %s`", nodes, clusterName) } } diff --git a/pkg/cluster/operation/scale_in.go b/pkg/cluster/operation/scale_in.go index b084bb4f4c..94d93cfa7d 100644 --- a/pkg/cluster/operation/scale_in.go +++ b/pkg/cluster/operation/scale_in.go @@ -315,13 +315,13 @@ func deleteMember( } case spec.ComponentDrainer: addr := instance.GetHost() + ":" + strconv.Itoa(instance.GetPort()) - err := binlogClient.OfflineDrainer(addr, addr) + err := binlogClient.OfflineDrainer(addr) if err != nil { - return errors.AddStack(err) + return err } case spec.ComponentPump: addr := instance.GetHost() + ":" + strconv.Itoa(instance.GetPort()) - err := binlogClient.OfflinePump(addr, addr) + err := binlogClient.OfflinePump(addr) if err != nil { return errors.AddStack(err) } diff --git a/pkg/cluster/spec/drainer.go b/pkg/cluster/spec/drainer.go index dc22666ab9..ea2a50cb8f 100644 --- a/pkg/cluster/spec/drainer.go +++ b/pkg/cluster/spec/drainer.go @@ -147,8 +147,13 @@ func (i *DrainerInstance) InitConfig( enableTLS := topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(DrainerSpec) + nodeID := i.GetHost() + ":" + strconv.Itoa(i.GetPort()) + // keep origin node id if is imported + if i.IsImported() { + nodeID = "" + } cfg := scripts.NewDrainerScript( - i.GetHost()+":"+strconv.Itoa(i.GetPort()), + nodeID, i.GetHost(), paths.Deploy, paths.Data[0], diff --git a/pkg/cluster/spec/pump.go b/pkg/cluster/spec/pump.go index 0a7e82cffc..dc87818f6b 100644 --- a/pkg/cluster/spec/pump.go +++ b/pkg/cluster/spec/pump.go @@ -146,8 +146,13 @@ func (i *PumpInstance) InitConfig( enableTLS := topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(PumpSpec) + nodeID := i.GetHost() + ":" + strconv.Itoa(i.GetPort()) + // keep origin node id if is imported + if i.IsImported() { + nodeID = "" + } cfg := scripts.NewPumpScript( - i.GetHost()+":"+strconv.Itoa(i.GetPort()), + nodeID, i.GetHost(), paths.Deploy, paths.Data[0], diff --git a/templates/scripts/run_drainer.sh.tpl b/templates/scripts/run_drainer.sh.tpl index 0d5a6b2993..2b5b7fca65 100644 --- a/templates/scripts/run_drainer.sh.tpl +++ b/templates/scripts/run_drainer.sh.tpl @@ -22,7 +22,9 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/drainer \ {{- else}} exec bin/drainer \ {{- end}} +{{- if .NodeID}} --node-id="{{.NodeID}}" \ +{{- end}} --addr="{{.IP}}:{{.Port}}" \ --pd-urls="{{template "PDList" .Endpoints}}" \ --data-dir="{{.DataDir}}" \ diff --git a/templates/scripts/run_pump.sh.tpl b/templates/scripts/run_pump.sh.tpl index 3109d524bf..323cb141e4 100644 --- a/templates/scripts/run_pump.sh.tpl +++ b/templates/scripts/run_pump.sh.tpl @@ -22,7 +22,9 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/pump \ {{- else}} exec bin/pump \ {{- end}} +{{- if .NodeID}} --node-id="{{.NodeID}}" \ +{{- end}} --addr="0.0.0.0:{{.Port}}" \ --advertise-addr="{{.Host}}:{{.Port}}" \ --pd-urls="{{template "PDList" .Endpoints}}" \