-
Notifications
You must be signed in to change notification settings - Fork 808
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support multiple Alertmanagers explicitly in the Ruler (#2851)
* Support Multiple Alertmanager URLs At the moment, the Ruler only support sending alerts to multiple Alertmanagers via DNS based service discovery. However, sending to multiple Alertmanager groups is something that Prometheus allows. I believe the Ruler should support this too and this commit introduces that. To keep backward compatibility we're reusing the same flag but allowing a list of 1+. With this, we can treat each URL as an Alertmanager group where multiple Alertmanagers per group is only supported if DNS service discovery is enabled. Signed-off-by: gotjosh <[email protected]> * Address review comments Signed-off-by: gotjosh <[email protected]> * Add the host check when service discovery is enabled We need to ensure address provided follow the SRV dns specification format. Signed-off-by: gotjosh <[email protected]> * It is actually space-separated Signed-off-by: gotjosh <[email protected]> * Update pkg/ruler/notifier.go Signed-off-by: Marco Pracucci <[email protected]> * Update pkg/ruler/notifier_test.go Signed-off-by: Marco Pracucci <[email protected]> Co-authored-by: Marco Pracucci <[email protected]>
- Loading branch information
Showing
5 changed files
with
268 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,190 @@ | ||
| package ruler | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
| "time" | ||
|
|
||
| config_util "github.com/prometheus/common/config" | ||
| "github.com/prometheus/common/model" | ||
| "github.com/prometheus/prometheus/config" | ||
| sd_config "github.com/prometheus/prometheus/discovery/config" | ||
| "github.com/prometheus/prometheus/discovery/dns" | ||
| "github.com/prometheus/prometheus/discovery/targetgroup" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestBuildNotifierConfig(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| cfg *Config | ||
| ncfg *config.Config | ||
| err error | ||
| }{ | ||
| { | ||
| name: "with no valid hosts, returns an empty config", | ||
| cfg: &Config{}, | ||
| ncfg: &config.Config{}, | ||
| }, | ||
| { | ||
| name: "with a single URL and no service discovery", | ||
| cfg: &Config{ | ||
| AlertmanagerURL: []string{"http://alertmanager.default.svc.cluster.local/alertmanager"}, | ||
| }, | ||
| ncfg: &config.Config{ | ||
| AlertingConfig: config.AlertingConfig{ | ||
| AlertmanagerConfigs: []*config.AlertmanagerConfig{ | ||
| { | ||
| APIVersion: "v1", | ||
| Scheme: "http", | ||
| PathPrefix: "/alertmanager", | ||
| ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{StaticConfigs: []*targetgroup.Group{{ | ||
| Targets: []model.LabelSet{{"__address__": "alertmanager.default.svc.cluster.local"}}, | ||
| }}}, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| name: "with a single URL and service discovery", | ||
| cfg: &Config{ | ||
| AlertmanagerURL: []string{"http://_http._tcp.alertmanager.default.svc.cluster.local/alertmanager"}, | ||
| AlertmanagerDiscovery: true, | ||
| AlertmanagerRefreshInterval: time.Duration(60), | ||
| }, | ||
| ncfg: &config.Config{ | ||
| AlertingConfig: config.AlertingConfig{ | ||
| AlertmanagerConfigs: []*config.AlertmanagerConfig{ | ||
| { | ||
| APIVersion: "v1", | ||
| Scheme: "http", | ||
| PathPrefix: "/alertmanager", | ||
| ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{DNSSDConfigs: []*dns.SDConfig{{ | ||
| Names: []string{"_http._tcp.alertmanager.default.svc.cluster.local"}, | ||
| RefreshInterval: 60, | ||
| Type: "SRV", | ||
| Port: 0, | ||
| }}}, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| name: "with service discovery and an invalid URL", | ||
| cfg: &Config{ | ||
| AlertmanagerURL: []string{"http://_http.default.svc.cluster.local/alertmanager"}, | ||
| AlertmanagerDiscovery: true, | ||
| }, | ||
| err: fmt.Errorf("when alertmanager-discovery is on, host name must be of the form _portname._tcp.service.fqdn (is \"alertmanager.default.svc.cluster.local\")"), | ||
| }, | ||
| { | ||
| name: "with multiple URLs and no service discovery", | ||
| cfg: &Config{ | ||
| AlertmanagerURL: []string{ | ||
| "http://alertmanager-0.default.svc.cluster.local/alertmanager", | ||
| "http://alertmanager-1.default.svc.cluster.local/alertmanager", | ||
| }, | ||
| }, | ||
| ncfg: &config.Config{ | ||
| AlertingConfig: config.AlertingConfig{ | ||
| AlertmanagerConfigs: []*config.AlertmanagerConfig{ | ||
| { | ||
| APIVersion: "v1", | ||
| Scheme: "http", | ||
| PathPrefix: "/alertmanager", | ||
| ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{StaticConfigs: []*targetgroup.Group{{ | ||
| Targets: []model.LabelSet{{"__address__": "alertmanager-0.default.svc.cluster.local"}}, | ||
| }}}, | ||
| }, | ||
| { | ||
| APIVersion: "v1", | ||
| Scheme: "http", | ||
| PathPrefix: "/alertmanager", | ||
| ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{StaticConfigs: []*targetgroup.Group{{ | ||
| Targets: []model.LabelSet{{"__address__": "alertmanager-1.default.svc.cluster.local"}}, | ||
| }}}, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| name: "with multiple URLs and service discovery", | ||
| cfg: &Config{ | ||
| AlertmanagerURL: []string{ | ||
| "http://_http._tcp.alertmanager-0.default.svc.cluster.local/alertmanager", | ||
| "http://_http._tcp.alertmanager-1.default.svc.cluster.local/alertmanager", | ||
| }, | ||
| AlertmanagerDiscovery: true, | ||
| AlertmanagerRefreshInterval: time.Duration(60), | ||
| }, | ||
| ncfg: &config.Config{ | ||
| AlertingConfig: config.AlertingConfig{ | ||
| AlertmanagerConfigs: []*config.AlertmanagerConfig{ | ||
| { | ||
| APIVersion: "v1", | ||
| Scheme: "http", | ||
| PathPrefix: "/alertmanager", | ||
| ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{DNSSDConfigs: []*dns.SDConfig{{ | ||
| Names: []string{"_http._tcp.alertmanager-0.default.svc.cluster.local"}, | ||
| RefreshInterval: 60, | ||
| Type: "SRV", | ||
| Port: 0, | ||
| }}}, | ||
| }, | ||
| { | ||
| APIVersion: "v1", | ||
| Scheme: "http", | ||
| PathPrefix: "/alertmanager", | ||
| ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{DNSSDConfigs: []*dns.SDConfig{{ | ||
| Names: []string{"_http._tcp.alertmanager-1.default.svc.cluster.local"}, | ||
| RefreshInterval: 60, | ||
| Type: "SRV", | ||
| Port: 0, | ||
| }}}, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| name: "with Basic Authentication", | ||
| cfg: &Config{ | ||
| AlertmanagerURL: []string{ | ||
| "http://marco:[email protected]/alertmanager", | ||
| }, | ||
| }, | ||
| ncfg: &config.Config{ | ||
| AlertingConfig: config.AlertingConfig{ | ||
| AlertmanagerConfigs: []*config.AlertmanagerConfig{ | ||
| { | ||
| HTTPClientConfig: config_util.HTTPClientConfig{ | ||
| BasicAuth: &config_util.BasicAuth{Username: "marco", Password: "hunter2"}, | ||
| }, | ||
| APIVersion: "v1", | ||
| Scheme: "http", | ||
| PathPrefix: "/alertmanager", | ||
| ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{StaticConfigs: []*targetgroup.Group{{ | ||
| Targets: []model.LabelSet{{"__address__": "alertmanager-0.default.svc.cluster.local"}}, | ||
| }}}, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| ncfg, err := buildNotifierConfig(tt.cfg) | ||
| if tt.err == nil { | ||
| require.NoError(t, err) | ||
| require.Equal(t, tt.ncfg, ncfg) | ||
| } else { | ||
| require.Error(t, tt.err, err) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters