Merge branch 'master' into refine-list-display-commands

pkg/cluster/spec/prometheus.go

@@ -40,13 +40,20 @@ type PrometheusSpec struct {
 	DataDir         string               `yaml:"data_dir,omitempty"`
 	LogDir          string               `yaml:"log_dir,omitempty"`
 	NumaNode        string               `yaml:"numa_node,omitempty" validate:"numa_node:editable"`
+	RemoteConfig    Remote               `yaml:"remote_config,omitempty" validate:"remote_config:ignore"`
 	Retention       string               `yaml:"storage_retention,omitempty" validate:"storage_retention:editable"`
 	ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
 	Arch            string               `yaml:"arch,omitempty"`
 	OS              string               `yaml:"os,omitempty"`
 	RuleDir         string               `yaml:"rule_dir,omitempty" validate:"rule_dir:editable"`
 }
 
+// Remote prometheus remote config
+type Remote struct {
+	RemoteWrite []map[string]interface{} `yaml:"remote_write,omitempty" validate:"remote_write:ignore"`
+	RemoteRead  []map[string]interface{} `yaml:"remote_read,omitempty" validate:"remote_read:ignore"`
+}
+
 // Role returns the component role of the instance
 func (s PrometheusSpec) Role() string {
 	return ComponentPrometheus
@@ -245,6 +252,7 @@ func (i *MonitorInstance) InitConfig(
 			cfig.AddDMWorker(host, uint64(port))
 		}
 	}
+
 	if monitoredOptions != nil {
 		for host := range uniqueHosts {
 			cfig.AddNodeExpoertor(host, uint64(monitoredOptions.NodeExporterPort))
@@ -253,6 +261,12 @@ func (i *MonitorInstance) InitConfig(
 		}
 	}
 
+	remoteCfg, err := encodeRemoteCfg2Yaml(spec.RemoteConfig)
+	if err != nil {
+		return err
+	}
+	cfig.SetRemoteConfig(string(remoteCfg))
+
 	if spec.RuleDir != "" {
 		filter := func(name string) bool { return strings.HasSuffix(name, ".rules.yml") }
 		err := i.IteratorLocalConfigDir(ctx, spec.RuleDir, filter, func(name string) error {

pkg/cluster/spec/server_config.go

@@ -30,6 +30,7 @@ import (
 	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/pingcap/tiup/pkg/utils"
+	"gopkg.in/yaml.v2"
 )
 
 const (
@@ -218,6 +219,20 @@ func merge2Toml(comp string, global, overwrite map[string]interface{}) ([]byte,
 	return buf.Bytes(), nil
 }
 
+func encodeRemoteCfg2Yaml(remote Remote) ([]byte, error) {
+	if len(remote.RemoteRead) == 0 && len(remote.RemoteWrite) == 0 {
+		return []byte{}, nil
+	}
+
+	buf := bytes.NewBufferString("")
+	enc := yaml.NewEncoder(buf)
+	err := enc.Encode(remote)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
 func mergeImported(importConfig []byte, specConfigs ...map[string]interface{}) (map[string]interface{}, error) {
 	var configData map[string]interface{}
 	if err := toml.Unmarshal(importConfig, &configData); err != nil {

pkg/cluster/spec/server_config_test.go

@@ -130,3 +130,42 @@ func (s *configSuite) TestFoldMap(c *check.C) {
 		},
 	})
 }
+
+func (s *configSuite) TestEncodeRemoteCfg(c *check.C) {
+	yamlData := []byte(`remote_write:
+- queue_config:
+    batch_send_deadline: 5m
+    capacity: 100000
+    max_samples_per_send: 10000
+    max_shards: 300
+  url: http://127.0.0.1:/8086/write
+remote_read:
+- url: http://127.0.0.1:/8086/read
+- url: http://127.0.0.1:/8087/read
+`)
+
+	bs, err := encodeRemoteCfg2Yaml(Remote{
+		RemoteWrite: []map[string]interface{}{
+			{
+				"url": "http://127.0.0.1:/8086/write",
+				"queue_config": map[string]interface{}{
+					"batch_send_deadline":  "5m",
+					"capacity":             100000,
+					"max_samples_per_send": 10000,
+					"max_shards":           300,
+				},
+			},
+		},
+		RemoteRead: []map[string]interface{}{
+			{
+				"url": "http://127.0.0.1:/8086/read",
+			},
+			{
+				"url": "http://127.0.0.1:/8087/read",
+			},
+		},
+	})
+
+	c.Assert(err, check.IsNil)
+	c.Assert(bs, check.BytesEquals, yamlData)
+}

pkg/cluster/template/config/prometheus.go

@@ -50,7 +50,8 @@ type PrometheusConfig struct {
 	DMMasterAddrs []string
 	DMWorkerAddrs []string
 
-	LocalRules []string
+	LocalRules   []string
+	RemoteConfig string
 }
 
 // NewPrometheusConfig returns a PrometheusConfig
@@ -193,6 +194,12 @@ func (c *PrometheusConfig) AddLocalRule(rule string) *PrometheusConfig {
 	return c
 }
 
+// SetRemoteConfig set remote read/write config
+func (c *PrometheusConfig) SetRemoteConfig(cfg string) *PrometheusConfig {
+	c.RemoteConfig = cfg
+	return c
+}
+
 // Config generate the config file data.
 func (c *PrometheusConfig) Config() ([]byte, error) {
 	fp := path.Join("/templates", "config", "prometheus.yml.tpl")

pkg/repository/mirror.go

@@ -493,6 +493,9 @@ func (l *httpMirror) Download(resource, targetDir string) error {
 			return nil
 		}
 		return r.Close()
+	}, utils.RetryOption{
+		Timeout:  time.Hour,
+		Attempts: 3,
 	})
 	if err != nil {
 		return err

templates/config/prometheus.yml.tpl

@@ -382,3 +382,7 @@ scrape_configs:
       - '{{.}}'
     {{- end}}
 {{- end}}
+
+{{- if .RemoteConfig}}
+{{.RemoteConfig}}
+{{- end}}