From c8e808758c3be98e62f9a8b006b65efc5937adf9 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 28 Jul 2021 11:29:46 +0800 Subject: [PATCH 1/2] examples: update monitored section of dm --- embed/templates/examples/dm/topology.example.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/embed/templates/examples/dm/topology.example.yaml b/embed/templates/examples/dm/topology.example.yaml index ae2be9af35..f17b5cf42a 100644 --- a/embed/templates/examples/dm/topology.example.yaml +++ b/embed/templates/examples/dm/topology.example.yaml @@ -70,3 +70,10 @@ alertmanager_servers: # deploy_dir: "/tidb-deploy/alertmanager-9093" # data_dir: "/tidb-data/alertmanager-9093" # log_dir: "/tidb-deploy/alertmanager-9093/log" + +# if monitored is set, node_exporter and blackbox_exporter will be +# deployed with the port specified, otherwise they are not deployed +# on the server to avoid conflict with tidb clusters +#monitored: +# node_exporter_port: 9100 +# blackbox_exporter_port: 9115 From 9172e076a6261da3279d47b005f62a81e75a901b Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 28 Jul 2021 11:42:53 +0800 Subject: [PATCH 2/2] embed: update dm topo & adjust dir layout --- components/cluster/command/template.go | 4 ++-- components/dm/command/template.go | 4 ++-- embed/embed.go | 12 +++++++++-- embed/embed_test.go | 21 ++++++++++++++++++- .../examples => examples/cluster}/local.yaml | 0 .../cluster}/minimal.yaml | 0 .../cluster}/multi-dc.yaml | 0 .../cluster}/topology.example.yaml | 0 .../{templates => }/examples/dm/minimal.yaml | 0 .../examples/dm/topology.example.yaml | 0 pkg/cluster/template/config/alertmanager.go | 2 +- pkg/cluster/template/config/blackbox.go | 2 +- pkg/cluster/template/config/config.go | 2 +- pkg/cluster/template/config/dashboard.go | 2 +- pkg/cluster/template/config/datasource.go | 2 +- pkg/cluster/template/config/grafana.go | 2 +- pkg/cluster/template/config/prometheus.go | 2 +- pkg/cluster/template/config/tispark.go | 2 +- pkg/cluster/template/scripts/alertmanager.go | 2 +- .../template/scripts/blackbox_exporter.go | 2 +- pkg/cluster/template/scripts/cdc.go | 2 +- pkg/cluster/template/scripts/dm_master.go | 4 ++-- pkg/cluster/template/scripts/dm_worker.go | 2 +- pkg/cluster/template/scripts/drainer.go | 2 +- pkg/cluster/template/scripts/grafana.go | 2 +- pkg/cluster/template/scripts/node_exporter.go | 2 +- pkg/cluster/template/scripts/pd.go | 2 +- pkg/cluster/template/scripts/prometheus.go | 2 +- pkg/cluster/template/scripts/pump.go | 2 +- pkg/cluster/template/scripts/scripts.go | 2 +- pkg/cluster/template/scripts/tidb.go | 2 +- pkg/cluster/template/scripts/tiflash.go | 2 +- pkg/cluster/template/scripts/tikv.go | 2 +- pkg/cluster/template/systemd/system.go | 2 +- pkg/cluster/template/systemd/tispark.go | 2 +- 35 files changed, 60 insertions(+), 33 deletions(-) rename embed/{templates/examples => examples/cluster}/local.yaml (100%) rename embed/{templates/examples => examples/cluster}/minimal.yaml (100%) rename embed/{templates/examples => examples/cluster}/multi-dc.yaml (100%) rename embed/{templates/examples => examples/cluster}/topology.example.yaml (100%) rename embed/{templates => }/examples/dm/minimal.yaml (100%) rename embed/{templates => }/examples/dm/topology.example.yaml (100%) diff --git a/components/cluster/command/template.go b/components/cluster/command/template.go index 979359c0de..707739dcec 100644 --- a/components/cluster/command/template.go +++ b/components/cluster/command/template.go @@ -61,8 +61,8 @@ func newTemplateCmd() *cobra.Command { name = "local.yaml" } - fp := path.Join("templates", "examples", name) - tpl, err := embed.ReadFile(fp) + fp := path.Join("examples", "cluster", name) + tpl, err := embed.ReadExample(fp) if err != nil { return err } diff --git a/components/dm/command/template.go b/components/dm/command/template.go index df4f325423..6cba6212ee 100644 --- a/components/dm/command/template.go +++ b/components/dm/command/template.go @@ -38,8 +38,8 @@ func newTemplateCmd() *cobra.Command { name = "topology.example.yaml" } - fp := path.Join("templates", "examples", "dm", name) - tpl, err := embed.ReadFile(fp) + fp := path.Join("examples", "dm", name) + tpl, err := embed.ReadExample(fp) if err != nil { return err } diff --git a/embed/embed.go b/embed/embed.go index 8e149bad25..3a75372ca8 100644 --- a/embed/embed.go +++ b/embed/embed.go @@ -7,7 +7,15 @@ import ( //go:embed templates var embededFiles goembed.FS -// ReadFile read the file embed. -func ReadFile(path string) ([]byte, error) { +// ReadTemplate read the template file embed. +func ReadTemplate(path string) ([]byte, error) { return embededFiles.ReadFile(path) } + +//go:embed examples +var embedExamples goembed.FS + +// ReadExample read an example file +func ReadExample(path string) ([]byte, error) { + return embedExamples.ReadFile(path) +} diff --git a/embed/embed_test.go b/embed/embed_test.go index 5522ec844e..048069de7d 100644 --- a/embed/embed_test.go +++ b/embed/embed_test.go @@ -47,7 +47,26 @@ func (s *embedSuite) TestCanReadTemplates(c *check.C) { data, err := os.ReadFile(path) c.Assert(err, check.IsNil) - embedData, err := ReadFile(path) + embedData, err := ReadTemplate(path) + c.Assert(err, check.IsNil) + + c.Assert(embedData, check.BytesEquals, data) + } +} + +// Test can read all file in /examples +func (s *embedSuite) TestCanReadExamples(c *check.C) { + paths, err := getAllFilePaths("examples") + c.Assert(err, check.IsNil) + c.Assert(len(paths), check.Greater, 0) + + for _, path := range paths { + c.Log("check file: ", path) + + data, err := os.ReadFile(path) + c.Assert(err, check.IsNil) + + embedData, err := ReadExample(path) c.Assert(err, check.IsNil) c.Assert(embedData, check.BytesEquals, data) diff --git a/embed/templates/examples/local.yaml b/embed/examples/cluster/local.yaml similarity index 100% rename from embed/templates/examples/local.yaml rename to embed/examples/cluster/local.yaml diff --git a/embed/templates/examples/minimal.yaml b/embed/examples/cluster/minimal.yaml similarity index 100% rename from embed/templates/examples/minimal.yaml rename to embed/examples/cluster/minimal.yaml diff --git a/embed/templates/examples/multi-dc.yaml b/embed/examples/cluster/multi-dc.yaml similarity index 100% rename from embed/templates/examples/multi-dc.yaml rename to embed/examples/cluster/multi-dc.yaml diff --git a/embed/templates/examples/topology.example.yaml b/embed/examples/cluster/topology.example.yaml similarity index 100% rename from embed/templates/examples/topology.example.yaml rename to embed/examples/cluster/topology.example.yaml diff --git a/embed/templates/examples/dm/minimal.yaml b/embed/examples/dm/minimal.yaml similarity index 100% rename from embed/templates/examples/dm/minimal.yaml rename to embed/examples/dm/minimal.yaml diff --git a/embed/templates/examples/dm/topology.example.yaml b/embed/examples/dm/topology.example.yaml similarity index 100% rename from embed/templates/examples/dm/topology.example.yaml rename to embed/examples/dm/topology.example.yaml diff --git a/pkg/cluster/template/config/alertmanager.go b/pkg/cluster/template/config/alertmanager.go index bb61e2f317..0b5bebbf0a 100644 --- a/pkg/cluster/template/config/alertmanager.go +++ b/pkg/cluster/template/config/alertmanager.go @@ -32,7 +32,7 @@ func NewAlertManagerConfig() *AlertManagerConfig { // Config generate the config file data. func (c *AlertManagerConfig) Config() ([]byte, error) { fp := path.Join("templates", "config", "alertmanager.yml") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/config/blackbox.go b/pkg/cluster/template/config/blackbox.go index 3d5ed728d7..d32db5aff5 100644 --- a/pkg/cluster/template/config/blackbox.go +++ b/pkg/cluster/template/config/blackbox.go @@ -39,7 +39,7 @@ func NewBlackboxConfig(deployDir string, tlsEnabled bool) *BlackboxConfig { // Config generate the config file data. func (c *BlackboxConfig) Config() ([]byte, error) { fp := path.Join("templates", "config", "blackbox.yml.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/config/config.go b/pkg/cluster/template/config/config.go index 4dfe007241..911b98fcee 100644 --- a/pkg/cluster/template/config/config.go +++ b/pkg/cluster/template/config/config.go @@ -22,5 +22,5 @@ import ( // GetConfig returns a raw config file from embed templates func GetConfig(filename string) ([]byte, error) { fp := filepath.Join("templates", "config", filename) - return embed.ReadFile(fp) + return embed.ReadTemplate(fp) } diff --git a/pkg/cluster/template/config/dashboard.go b/pkg/cluster/template/config/dashboard.go index fc2c0ac607..df1aa7020f 100644 --- a/pkg/cluster/template/config/dashboard.go +++ b/pkg/cluster/template/config/dashboard.go @@ -39,7 +39,7 @@ func NewDashboardConfig(cluster, deployDir string) *DashboardConfig { // Config generate the config file data. func (c *DashboardConfig) Config() ([]byte, error) { fp := path.Join("templates", "config", "dashboard.yml.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/config/datasource.go b/pkg/cluster/template/config/datasource.go index 2d22b6509a..d05683726f 100644 --- a/pkg/cluster/template/config/datasource.go +++ b/pkg/cluster/template/config/datasource.go @@ -47,7 +47,7 @@ func (c *DatasourceConfig) WithPort(port uint64) *DatasourceConfig { // Config generate the config file data. func (c *DatasourceConfig) Config() ([]byte, error) { fp := path.Join("templates", "config", "datasource.yml.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/config/grafana.go b/pkg/cluster/template/config/grafana.go index 00190ccaf7..e70178ab88 100644 --- a/pkg/cluster/template/config/grafana.go +++ b/pkg/cluster/template/config/grafana.go @@ -82,7 +82,7 @@ func (c *GrafanaConfig) WithDomain(domain string) *GrafanaConfig { // Config generate the config file data. func (c *GrafanaConfig) Config() ([]byte, error) { fp := path.Join("templates", "config", "grafana.ini.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/config/prometheus.go b/pkg/cluster/template/config/prometheus.go index 76e0611b00..cba903a515 100644 --- a/pkg/cluster/template/config/prometheus.go +++ b/pkg/cluster/template/config/prometheus.go @@ -211,7 +211,7 @@ func (c *PrometheusConfig) SetRemoteConfig(cfg string) *PrometheusConfig { // Config generate the config file data. func (c *PrometheusConfig) Config() ([]byte, error) { fp := path.Join("templates", "config", "prometheus.yml.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/config/tispark.go b/pkg/cluster/template/config/tispark.go index c173b278dd..e8427dd485 100644 --- a/pkg/cluster/template/config/tispark.go +++ b/pkg/cluster/template/config/tispark.go @@ -49,7 +49,7 @@ func (c *TiSparkConfig) WithCustomFields(m map[string]interface{}) *TiSparkConfi // Config generate the config file data. func (c *TiSparkConfig) Config() ([]byte, error) { fp := filepath.Join("templates", "config", "spark-defaults.conf.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/alertmanager.go b/pkg/cluster/template/scripts/alertmanager.go index b70a6cde46..ba6667dd23 100644 --- a/pkg/cluster/template/scripts/alertmanager.go +++ b/pkg/cluster/template/scripts/alertmanager.go @@ -84,7 +84,7 @@ func (c *AlertManagerScript) ConfigToFile(file string) error { // Config generate the config file data. func (c *AlertManagerScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_alertmanager.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/blackbox_exporter.go b/pkg/cluster/template/scripts/blackbox_exporter.go index 55c543d574..aa90bed955 100644 --- a/pkg/cluster/template/scripts/blackbox_exporter.go +++ b/pkg/cluster/template/scripts/blackbox_exporter.go @@ -54,7 +54,7 @@ func (c *BlackboxExporterScript) WithNumaNode(numa string) *BlackboxExporterScri // Config generate the config file data. func (c *BlackboxExporterScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_blackbox_exporter.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/cdc.go b/pkg/cluster/template/scripts/cdc.go index 77432ff441..7fe532397b 100644 --- a/pkg/cluster/template/scripts/cdc.go +++ b/pkg/cluster/template/scripts/cdc.go @@ -85,7 +85,7 @@ func (c *CDCScript) WithDataDirEnabled() *CDCScript { // Config generate the config file data. func (c *CDCScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_cdc.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/dm_master.go b/pkg/cluster/template/scripts/dm_master.go index 8df2939d1d..d0e52e8e58 100644 --- a/pkg/cluster/template/scripts/dm_master.go +++ b/pkg/cluster/template/scripts/dm_master.go @@ -91,7 +91,7 @@ func (c *DMMasterScript) AppendEndpoints(ends ...*DMMasterScript) *DMMasterScrip // Config generate the config file data. func (c *DMMasterScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_dm-master.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } @@ -174,7 +174,7 @@ func (c *DMMasterScaleScript) AppendEndpoints(ends ...*DMMasterScript) *DMMaster // Config generate the config file data. func (c *DMMasterScaleScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_dm-master_scale.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/dm_worker.go b/pkg/cluster/template/scripts/dm_worker.go index 1bd9e89d2b..1bdd1f6536 100644 --- a/pkg/cluster/template/scripts/dm_worker.go +++ b/pkg/cluster/template/scripts/dm_worker.go @@ -65,7 +65,7 @@ func (c *DMWorkerScript) AppendEndpoints(ends ...*DMMasterScript) *DMWorkerScrip // Config generate the config file data. func (c *DMWorkerScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_dm-worker.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/drainer.go b/pkg/cluster/template/scripts/drainer.go index 28fc0cd13a..ed2831965a 100644 --- a/pkg/cluster/template/scripts/drainer.go +++ b/pkg/cluster/template/scripts/drainer.go @@ -75,7 +75,7 @@ func (c *DrainerScript) AppendEndpoints(ends ...*PDScript) *DrainerScript { // Config generate the config file data. func (c *DrainerScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_drainer.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/grafana.go b/pkg/cluster/template/scripts/grafana.go index fc4d30fa54..a43089276b 100644 --- a/pkg/cluster/template/scripts/grafana.go +++ b/pkg/cluster/template/scripts/grafana.go @@ -57,7 +57,7 @@ func (c *GrafanaScript) Config() ([]byte, error) { fp = path.Join("templates", "scripts", "run_grafana.sh.tpl") } - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/node_exporter.go b/pkg/cluster/template/scripts/node_exporter.go index 393ea1b786..da82cbaeb0 100644 --- a/pkg/cluster/template/scripts/node_exporter.go +++ b/pkg/cluster/template/scripts/node_exporter.go @@ -54,7 +54,7 @@ func (c *NodeExporterScript) WithNumaNode(numa string) *NodeExporterScript { // Config generate the config file data. func (c *NodeExporterScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_node_exporter.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/pd.go b/pkg/cluster/template/scripts/pd.go index e462e5f0ec..16f3ae1876 100644 --- a/pkg/cluster/template/scripts/pd.go +++ b/pkg/cluster/template/scripts/pd.go @@ -124,7 +124,7 @@ func (c *PDScript) Config() ([]byte, error) { func (c *PDScript) configWithScript(script string) ([]byte, error) { fp := path.Join("templates", "scripts", script) - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/prometheus.go b/pkg/cluster/template/scripts/prometheus.go index 0c0f737c95..4dc7444116 100644 --- a/pkg/cluster/template/scripts/prometheus.go +++ b/pkg/cluster/template/scripts/prometheus.go @@ -82,7 +82,7 @@ func (c *PrometheusScript) Config() ([]byte, error) { fp = path.Join("templates", "scripts", "run_prometheus.sh.tpl") } - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/pump.go b/pkg/cluster/template/scripts/pump.go index f9eb91cc6a..8d368b6ccc 100644 --- a/pkg/cluster/template/scripts/pump.go +++ b/pkg/cluster/template/scripts/pump.go @@ -68,7 +68,7 @@ func (c *PumpScript) AppendEndpoints(ends ...*PDScript) *PumpScript { // Config generate the config file data. func (c *PumpScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_pump.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/scripts.go b/pkg/cluster/template/scripts/scripts.go index c29e2c2064..e95fe4bf66 100644 --- a/pkg/cluster/template/scripts/scripts.go +++ b/pkg/cluster/template/scripts/scripts.go @@ -22,5 +22,5 @@ import ( // GetScript returns a raw config file from embed templates func GetScript(filename string) ([]byte, error) { fp := filepath.Join("templates", "scripts", filename) - return embed.ReadFile(fp) + return embed.ReadTemplate(fp) } diff --git a/pkg/cluster/template/scripts/tidb.go b/pkg/cluster/template/scripts/tidb.go index 6f713e104e..6501b14d57 100644 --- a/pkg/cluster/template/scripts/tidb.go +++ b/pkg/cluster/template/scripts/tidb.go @@ -88,7 +88,7 @@ func (c *TiDBScript) AppendEndpoints(ends ...*PDScript) *TiDBScript { // Config generate the config file data. func (c *TiDBScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_tidb.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/tiflash.go b/pkg/cluster/template/scripts/tiflash.go index bcc28c20ff..db40634df1 100644 --- a/pkg/cluster/template/scripts/tiflash.go +++ b/pkg/cluster/template/scripts/tiflash.go @@ -121,7 +121,7 @@ func (c *TiFlashScript) AppendEndpoints(ends ...*PDScript) *TiFlashScript { // Config generate the config file data. func (c *TiFlashScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_tiflash.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/scripts/tikv.go b/pkg/cluster/template/scripts/tikv.go index aaac858982..33222da0e2 100644 --- a/pkg/cluster/template/scripts/tikv.go +++ b/pkg/cluster/template/scripts/tikv.go @@ -96,7 +96,7 @@ func (c *TiKVScript) AppendEndpoints(ends ...*PDScript) *TiKVScript { // Config generate the config file data. func (c *TiKVScript) Config() ([]byte, error) { fp := path.Join("templates", "scripts", "run_tikv.sh.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/systemd/system.go b/pkg/cluster/template/systemd/system.go index 51622b46cf..5bf4af1fd2 100644 --- a/pkg/cluster/template/systemd/system.go +++ b/pkg/cluster/template/systemd/system.go @@ -90,7 +90,7 @@ func (c *Config) ConfigToFile(file string) error { // Config generate the config file data. func (c *Config) Config() ([]byte, error) { fp := path.Join("templates", "systemd", "system.service.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err } diff --git a/pkg/cluster/template/systemd/tispark.go b/pkg/cluster/template/systemd/tispark.go index 3c18d1ba81..4646941306 100644 --- a/pkg/cluster/template/systemd/tispark.go +++ b/pkg/cluster/template/systemd/tispark.go @@ -61,7 +61,7 @@ func (c *TiSparkConfig) ConfigToFile(file string) error { // Config generate the config file data. func (c *TiSparkConfig) Config() ([]byte, error) { fp := path.Join("templates", "systemd", "tispark.service.tpl") - tpl, err := embed.ReadFile(fp) + tpl, err := embed.ReadTemplate(fp) if err != nil { return nil, err }