From f35424c6d0c504b1c6a6a681999235ca40d969fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Fri, 10 Mar 2023 13:07:54 +0100 Subject: [PATCH] Render metrics endpoint and metrics profile cfg (#268) * Render metrics endpoint and metrics profile cfg Signed-off-by: Raul Sevilla * Fix OCP wrapper scenario Signed-off-by: Raul Sevilla * Use env vars in alert-profile rendering Signed-off-by: Raul Sevilla --------- Signed-off-by: Raul Sevilla --- pkg/alerting/alert_manager.go | 4 +++- pkg/burner/job.go | 2 +- pkg/config/config.go | 23 ++++------------------- pkg/prometheus/prometheus.go | 8 +++++--- pkg/util/metrics/utils.go | 14 ++++++++++++-- pkg/util/template.go | 12 ++++++++++++ pkg/workloads/helpers.go | 12 ++++++++---- 7 files changed, 45 insertions(+), 30 deletions(-) diff --git a/pkg/alerting/alert_manager.go b/pkg/alerting/alert_manager.go index 44334d142..c43fff917 100644 --- a/pkg/alerting/alert_manager.go +++ b/pkg/alerting/alert_manager.go @@ -115,9 +115,11 @@ func (a *AlertManager) Evaluate(start, end time.Time) int { elapsed := int(end.Sub(start).Minutes()) var renderedQuery bytes.Buffer result := Passed + vars := util.EnvToMap() + vars["elapsed"] = fmt.Sprintf("%dm", elapsed) for _, alert := range a.alertProfile { t, _ := template.New("").Parse(alert.Expr) - t.Execute(&renderedQuery, map[string]string{"elapsed": fmt.Sprintf("%dm", elapsed)}) + t.Execute(&renderedQuery, vars) expr := renderedQuery.String() renderedQuery.Reset() log.Infof("Evaluating expression: '%s'", expr) diff --git a/pkg/burner/job.go b/pkg/burner/job.go index fc8915152..f4cc26247 100644 --- a/pkg/burner/job.go +++ b/pkg/burner/job.go @@ -171,7 +171,7 @@ func Run(configSpec config.Spec, uuid string, prometheusClients []*prometheus.Pr // Update end time of last job jobList[len(jobList)-1].End = time.Now().UTC() if len(prometheusClients) > 0 { - log.Infof("Waiting %v extra before starting to scraping prometheus endpoints", prometheusClients[0].Step) + log.Infof("Waiting %v extra before scraping prometheus endpoints", prometheusClients[0].Step) time.Sleep(prometheusClients[0].Step) } for idx, prometheusClient := range prometheusClients { diff --git a/pkg/config/config.go b/pkg/config/config.go index 5bd059ba1..9f5994ddd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,7 +21,6 @@ import ( "io" "os" "path/filepath" - "strings" "time" "github.com/cloud-bulldozer/kube-burner/log" @@ -52,23 +51,6 @@ var configSpec = Spec{ }, } -func envToMap() map[string]interface{} { - envMap := make(map[string]interface{}) - for _, v := range os.Environ() { - envVar := strings.SplitN(v, "=", 2) - envMap[envVar[0]] = envVar[1] - } - return envMap -} - -func renderConfig(cfg []byte) ([]byte, error) { - rendered, err := util.RenderTemplate(cfg, envToMap(), util.MissingKeyError) - if err != nil { - return rendered, fmt.Errorf("Error rendering configuration template: %s", err) - } - return rendered, nil -} - // UnmarshalYAML implements Unmarshaller to customize object defaults func (o *Object) UnmarshalYAML(unmarshal func(interface{}) error) error { type rawObject Object @@ -123,7 +105,10 @@ func Parse(c string, jobsRequired bool) (Spec, error) { if err != nil { return configSpec, fmt.Errorf("Error reading configuration file %s: %s", c, err) } - renderedCfg, err := renderConfig(cfg) + renderedCfg, err := util.RenderTemplate(cfg, util.EnvToMap(), util.MissingKeyError) + if err != nil { + return configSpec, fmt.Errorf("Error rendering configuration template: %s", err) + } if err != nil { return configSpec, err } diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go index 3b5b89445..9d6450a59 100644 --- a/pkg/prometheus/prometheus.go +++ b/pkg/prometheus/prometheus.go @@ -73,7 +73,7 @@ func NewPrometheusClient(configSpec config.Spec, url, token, username, password, } if configSpec.GlobalConfig.MetricsProfile != "" { if err := p.readProfile(configSpec.GlobalConfig.MetricsProfile); err != nil { - log.Fatal(err) + log.Fatalf("Metrics-profile error: %v", err.Error()) } } return &p, nil @@ -92,7 +92,7 @@ func (p *Prometheus) verifyConnection() error { func (p *Prometheus) readProfile(metricsProfile string) error { f, err := util.ReadConfig(metricsProfile) if err != nil { - log.Fatalf("Error reading metrics profile %s: %s", metricsProfile, err) + return fmt.Errorf("error reading metrics profile %s: %s", metricsProfile, err) } yamlDec := yaml.NewDecoder(f) yamlDec.KnownFields(true) @@ -110,11 +110,13 @@ func (p *Prometheus) ScrapeJobsMetrics(indexer *indexers.Indexer) error { var err error var v model.Value var renderedQuery bytes.Buffer + vars := util.EnvToMap() + vars["elapsed"] = fmt.Sprintf("%dm", elapsed) log.Infof("🔍 Scraping prometheus metrics for benchmark from %s to %s", start.Format(time.RFC3339), end.Format(time.RFC3339)) for _, md := range p.MetricProfile { var datapoints []interface{} t, _ := template.New("").Parse(md.Query) - t.Execute(&renderedQuery, map[string]string{"elapsed": fmt.Sprintf("%dm", elapsed)}) + t.Execute(&renderedQuery, vars) query := renderedQuery.String() renderedQuery.Reset() if md.Instant { diff --git a/pkg/util/metrics/utils.go b/pkg/util/metrics/utils.go index 4c65b8985..9ce8cbd6d 100644 --- a/pkg/util/metrics/utils.go +++ b/pkg/util/metrics/utils.go @@ -15,6 +15,8 @@ package metrics import ( + "bytes" + "io" "time" "github.com/cloud-bulldozer/kube-burner/log" @@ -45,7 +47,15 @@ func DecodeMetricsEndpoint(metricsEndpoint string, metricsEndpoints *[]prometheu if err != nil { log.Fatalf("Error reading metricsEndpoint %s: %s", metricsEndpoint, err) } - yamlDec := yaml.NewDecoder(f) + cfg, err := io.ReadAll(f) + if err != nil { + log.Fatalf("Error reading configuration file %s: %s", metricsEndpoint, err) + } + renderedME, err := util.RenderTemplate(cfg, util.EnvToMap(), util.MissingKeyError) + if err != nil { + log.Fatalf("Template error in %s: %s", metricsEndpoint, err) + } + yamlDec := yaml.NewDecoder(bytes.NewReader(renderedME)) yamlDec.KnownFields(true) if err := yamlDec.Decode(&metricsEndpoints); err != nil { log.Fatalf("Error decoding metricsEndpoint %s: %s", metricsEndpoint, err) @@ -54,7 +64,7 @@ func DecodeMetricsEndpoint(metricsEndpoint string, metricsEndpoints *[]prometheu // Scrapes prometheus metrics func ScrapeMetrics(p *prometheus.Prometheus, indexer *indexers.Indexer) { - log.Infof("Scraping for the prometheus entry with params - {Endpoint:%v, Profile:%v, Start:%v, End:%v}", + log.Infof("Scraping for the prometheus entry with params - Endpoint: %v, Profile: %v, Start: %v, End: %v", p.Endpoint, p.ConfigSpec.GlobalConfig.MetricsProfile, p.JobList[0].Start.Format(time.RFC3339), diff --git a/pkg/util/template.go b/pkg/util/template.go index 68c6c0b4c..793d653b6 100644 --- a/pkg/util/template.go +++ b/pkg/util/template.go @@ -18,6 +18,8 @@ import ( "bytes" "fmt" "math/rand" + "os" + "strings" "text/template" "time" @@ -83,3 +85,13 @@ func RenderTemplate(original []byte, inputData interface{}, options templateOpti log.Tracef("Rendered template: %s", rendered.String()) return rendered.Bytes(), nil } + +// EnvToMap returns the host environment variables as a map +func EnvToMap() map[string]interface{} { + envMap := make(map[string]interface{}) + for _, v := range os.Environ() { + envVar := strings.SplitN(v, "=", 2) + envMap[envVar[0]] = envVar[1] + } + return envMap +} diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index 99500632a..73c21f384 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -219,25 +219,29 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { metrics.DecodeMetricsEndpoint(wh.metricsEndpoint, &metricsEndpoints) } else { metricsEndpoints = append(metricsEndpoints, prometheus.MetricEndpoint{ - Endpoint: wh.prometheusURL, - Token: wh.prometheusToken, + Endpoint: wh.prometheusURL, + AlertProfile: alertsProfile, + Token: wh.prometheusToken, }) } for _, metricsEndpoint := range metricsEndpoints { // Updating the prometheus endpoint actually being used in spec. configSpec.GlobalConfig.PrometheusURL = metricsEndpoint.Endpoint + configSpec.GlobalConfig.MetricsProfile = metricsEndpoint.Profile + configSpec.GlobalConfig.AlertProfile = metricsEndpoint.AlertProfile p, err := prometheus.NewPrometheusClient(configSpec, metricsEndpoint.Endpoint, metricsEndpoint.Token, "", "", wh.Metadata.UUID, true, 30*time.Second, metadata) if err != nil { log.Fatal(err) } - if wh.alerting { - alertM, err = alerting.NewAlertManager(alertsProfile, wh.Metadata.UUID, configSpec.GlobalConfig.IndexerConfig.DefaultIndex, indexer, p) + if wh.alerting && configSpec.GlobalConfig.AlertProfile != "" { + alertM, err = alerting.NewAlertManager(configSpec.GlobalConfig.AlertProfile, wh.Metadata.UUID, configSpec.GlobalConfig.IndexerConfig.DefaultIndex, indexer, p) if err != nil { log.Fatal(err) } } prometheusClients = append(prometheusClients, p) alertMs = append(alertMs, alertM) + alertM = nil } rc, err = burner.Run(configSpec, wh.Metadata.UUID, prometheusClients, alertMs, indexer, wh.timeout, metadata) if err != nil {