Render metrics endpoint and metrics profile cfg (kube-burner#268)

* Render metrics endpoint and metrics profile cfg Signed-off-by: Raul Sevilla <[email protected]> * Fix OCP wrapper scenario Signed-off-by: Raul Sevilla <[email protected]> * Use env vars in alert-profile rendering Signed-off-by: Raul Sevilla <[email protected]> --------- Signed-off-by: Raul Sevilla <[email protected]>
rsevilla87 · Mar 10, 2023 · f35424c · f35424c
1 parent 4526d57
commit f35424c
Show file tree

Hide file tree

Showing 7 changed files with 45 additions and 30 deletions.
diff --git a/pkg/alerting/alert_manager.go b/pkg/alerting/alert_manager.go
@@ -115,9 +115,11 @@ func (a *AlertManager) Evaluate(start, end time.Time) int {
 	elapsed := int(end.Sub(start).Minutes())
 	var renderedQuery bytes.Buffer
 	result := Passed
+	vars := util.EnvToMap()
+	vars["elapsed"] = fmt.Sprintf("%dm", elapsed)
 	for _, alert := range a.alertProfile {
 		t, _ := template.New("").Parse(alert.Expr)
-		t.Execute(&renderedQuery, map[string]string{"elapsed": fmt.Sprintf("%dm", elapsed)})
+		t.Execute(&renderedQuery, vars)
 		expr := renderedQuery.String()
 		renderedQuery.Reset()
 		log.Infof("Evaluating expression: '%s'", expr)

diff --git a/pkg/burner/job.go b/pkg/burner/job.go
@@ -171,7 +171,7 @@ func Run(configSpec config.Spec, uuid string, prometheusClients []*prometheus.Pr
 		// Update end time of last job
 		jobList[len(jobList)-1].End = time.Now().UTC()
 		if len(prometheusClients) > 0 {
-			log.Infof("Waiting %v extra before starting to scraping prometheus endpoints", prometheusClients[0].Step)
+			log.Infof("Waiting %v extra before scraping prometheus endpoints", prometheusClients[0].Step)
 			time.Sleep(prometheusClients[0].Step)
 		}
 		for idx, prometheusClient := range prometheusClients {

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -21,7 +21,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/cloud-bulldozer/kube-burner/log"
@@ -52,23 +51,6 @@ var configSpec = Spec{
 	},
 }
 
-func envToMap() map[string]interface{} {
-	envMap := make(map[string]interface{})
-	for _, v := range os.Environ() {
-		envVar := strings.SplitN(v, "=", 2)
-		envMap[envVar[0]] = envVar[1]
-	}
-	return envMap
-}
-
-func renderConfig(cfg []byte) ([]byte, error) {
-	rendered, err := util.RenderTemplate(cfg, envToMap(), util.MissingKeyError)
-	if err != nil {
-		return rendered, fmt.Errorf("Error rendering configuration template: %s", err)
-	}
-	return rendered, nil
-}
-
 // UnmarshalYAML implements Unmarshaller to customize object defaults
 func (o *Object) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawObject Object
@@ -123,7 +105,10 @@ func Parse(c string, jobsRequired bool) (Spec, error) {
 	if err != nil {
 		return configSpec, fmt.Errorf("Error reading configuration file %s: %s", c, err)
 	}
-	renderedCfg, err := renderConfig(cfg)
+	renderedCfg, err := util.RenderTemplate(cfg, util.EnvToMap(), util.MissingKeyError)
+	if err != nil {
+		return configSpec, fmt.Errorf("Error rendering configuration template: %s", err)
+	}
 	if err != nil {
 		return configSpec, err
 	}

diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go
@@ -73,7 +73,7 @@ func NewPrometheusClient(configSpec config.Spec, url, token, username, password,
 	}
 	if configSpec.GlobalConfig.MetricsProfile != "" {
 		if err := p.readProfile(configSpec.GlobalConfig.MetricsProfile); err != nil {
-			log.Fatal(err)
+			log.Fatalf("Metrics-profile error: %v", err.Error())
 		}
 	}
 	return &p, nil
@@ -92,7 +92,7 @@ func (p *Prometheus) verifyConnection() error {
 func (p *Prometheus) readProfile(metricsProfile string) error {
 	f, err := util.ReadConfig(metricsProfile)
 	if err != nil {
-		log.Fatalf("Error reading metrics profile %s: %s", metricsProfile, err)
+		return fmt.Errorf("error reading metrics profile %s: %s", metricsProfile, err)
 	}
 	yamlDec := yaml.NewDecoder(f)
 	yamlDec.KnownFields(true)
@@ -110,11 +110,13 @@ func (p *Prometheus) ScrapeJobsMetrics(indexer *indexers.Indexer) error {
 	var err error
 	var v model.Value
 	var renderedQuery bytes.Buffer
+	vars := util.EnvToMap()
+	vars["elapsed"] = fmt.Sprintf("%dm", elapsed)
 	log.Infof("🔍 Scraping prometheus metrics for benchmark from %s to %s", start.Format(time.RFC3339), end.Format(time.RFC3339))
 	for _, md := range p.MetricProfile {
 		var datapoints []interface{}
 		t, _ := template.New("").Parse(md.Query)
-		t.Execute(&renderedQuery, map[string]string{"elapsed": fmt.Sprintf("%dm", elapsed)})
+		t.Execute(&renderedQuery, vars)
 		query := renderedQuery.String()
 		renderedQuery.Reset()
 		if md.Instant {

diff --git a/pkg/util/metrics/utils.go b/pkg/util/metrics/utils.go
@@ -15,6 +15,8 @@
 package metrics
 
 import (
+	"bytes"
+	"io"
 	"time"
 
 	"github.com/cloud-bulldozer/kube-burner/log"
@@ -45,7 +47,15 @@ func DecodeMetricsEndpoint(metricsEndpoint string, metricsEndpoints *[]prometheu
 	if err != nil {
 		log.Fatalf("Error reading metricsEndpoint %s: %s", metricsEndpoint, err)
 	}
-	yamlDec := yaml.NewDecoder(f)
+	cfg, err := io.ReadAll(f)
+	if err != nil {
+		log.Fatalf("Error reading configuration file %s: %s", metricsEndpoint, err)
+	}
+	renderedME, err := util.RenderTemplate(cfg, util.EnvToMap(), util.MissingKeyError)
+	if err != nil {
+		log.Fatalf("Template error in %s: %s", metricsEndpoint, err)
+	}
+	yamlDec := yaml.NewDecoder(bytes.NewReader(renderedME))
 	yamlDec.KnownFields(true)
 	if err := yamlDec.Decode(&metricsEndpoints); err != nil {
 		log.Fatalf("Error decoding metricsEndpoint %s: %s", metricsEndpoint, err)
@@ -54,7 +64,7 @@ func DecodeMetricsEndpoint(metricsEndpoint string, metricsEndpoints *[]prometheu
 
 // Scrapes prometheus metrics
 func ScrapeMetrics(p *prometheus.Prometheus, indexer *indexers.Indexer) {
-	log.Infof("Scraping for the prometheus entry with params - {Endpoint:%v, Profile:%v, Start:%v, End:%v}",
+	log.Infof("Scraping for the prometheus entry with params - Endpoint: %v, Profile: %v, Start: %v, End: %v",
 		p.Endpoint,
 		p.ConfigSpec.GlobalConfig.MetricsProfile,
 		p.JobList[0].Start.Format(time.RFC3339),

diff --git a/pkg/util/template.go b/pkg/util/template.go
@@ -18,6 +18,8 @@ import (
 	"bytes"
 	"fmt"
 	"math/rand"
+	"os"
+	"strings"
 	"text/template"
 	"time"
 
@@ -83,3 +85,13 @@ func RenderTemplate(original []byte, inputData interface{}, options templateOpti
 	log.Tracef("Rendered template: %s", rendered.String())
 	return rendered.Bytes(), nil
 }
+
+// EnvToMap returns the host environment variables as a map
+func EnvToMap() map[string]interface{} {
+	envMap := make(map[string]interface{})
+	for _, v := range os.Environ() {
+		envVar := strings.SplitN(v, "=", 2)
+		envMap[envVar[0]] = envVar[1]
+	}
+	return envMap
+}
diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go
@@ -219,25 +219,29 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) {
 		metrics.DecodeMetricsEndpoint(wh.metricsEndpoint, &metricsEndpoints)
 	} else {
 		metricsEndpoints = append(metricsEndpoints, prometheus.MetricEndpoint{
-			Endpoint: wh.prometheusURL,
-			Token:    wh.prometheusToken,
+			Endpoint:     wh.prometheusURL,
+			AlertProfile: alertsProfile,
+			Token:        wh.prometheusToken,
 		})
 	}
 	for _, metricsEndpoint := range metricsEndpoints {
 		// Updating the prometheus endpoint actually being used in spec.
 		configSpec.GlobalConfig.PrometheusURL = metricsEndpoint.Endpoint
+		configSpec.GlobalConfig.MetricsProfile = metricsEndpoint.Profile
+		configSpec.GlobalConfig.AlertProfile = metricsEndpoint.AlertProfile
 		p, err := prometheus.NewPrometheusClient(configSpec, metricsEndpoint.Endpoint, metricsEndpoint.Token, "", "", wh.Metadata.UUID, true, 30*time.Second, metadata)
 		if err != nil {
 			log.Fatal(err)
 		}
-		if wh.alerting {
-			alertM, err = alerting.NewAlertManager(alertsProfile, wh.Metadata.UUID, configSpec.GlobalConfig.IndexerConfig.DefaultIndex, indexer, p)
+		if wh.alerting && configSpec.GlobalConfig.AlertProfile != "" {
+			alertM, err = alerting.NewAlertManager(configSpec.GlobalConfig.AlertProfile, wh.Metadata.UUID, configSpec.GlobalConfig.IndexerConfig.DefaultIndex, indexer, p)
 			if err != nil {
 				log.Fatal(err)
 			}
 		}
 		prometheusClients = append(prometheusClients, p)
 		alertMs = append(alertMs, alertM)
+		alertM = nil
 	}
 	rc, err = burner.Run(configSpec, wh.Metadata.UUID, prometheusClients, alertMs, indexer, wh.timeout, metadata)
 	if err != nil {