Add additional metadata to metrics (kube-burner#253)

- Add JobConfig metadata To Do : - Add Extra Metadata from ocp wrapper Note: Ignore JobConfig.Objects Signed-off-by: Joe Talerico <[email protected]>
rsevilla87 · Feb 17, 2023 · 3635a48 · 3635a48
1 parent df552e0
commit 3635a48
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 20 deletions.
diff --git a/pkg/burner/job.go b/pkg/burner/job.go
@@ -148,6 +148,7 @@ func Run(configSpec config.Spec, uuid string, p *prometheus.Prometheus, alertM *
 			}
 			jobList[jobPosition].End = time.Now().UTC()
 			prometheusJob.End = jobList[jobPosition].End
+			prometheusJob.JobConfig = job.Config
 			elapsedTime := prometheusJob.End.Sub(prometheusJob.Start).Seconds()
 			// Don't append to Prometheus jobList when prometheus it's not initialized
 			if p != nil {

diff --git a/pkg/measurements/metrics/metrics.go b/pkg/measurements/metrics/metrics.go
@@ -23,21 +23,23 @@ import (
 	"time"
 
 	"github.com/cloud-bulldozer/kube-burner/log"
+	"github.com/cloud-bulldozer/kube-burner/pkg/config"
 	"github.com/cloud-bulldozer/kube-burner/pkg/measurements/types"
 )
 
 // LatencyQuantiles holds the latency measurement quantiles
 type LatencyQuantiles struct {
-	QuantileName string    `json:"quantileName"`
-	UUID         string    `json:"uuid"`
-	P99          int       `json:"P99"`
-	P95          int       `json:"P95"`
-	P50          int       `json:"P50"`
-	Max          int       `json:"max"`
-	Avg          int       `json:"avg"`
-	Timestamp    time.Time `json:"timestamp"`
-	MetricName   string    `json:"metricName"`
-	JobName      string    `json:"jobName"`
+	QuantileName string     `json:"quantileName"`
+	UUID         string     `json:"uuid"`
+	P99          int        `json:"P99"`
+	P95          int        `json:"P95"`
+	P50          int        `json:"P50"`
+	Max          int        `json:"max"`
+	Avg          int        `json:"avg"`
+	Timestamp    time.Time  `json:"timestamp"`
+	MetricName   string     `json:"metricName"`
+	JobName      string     `json:"jobName"`
+	JobConfig    config.Job `json:"jobConfig"`
 }
 
 // SetQuantile adds quantile value

diff --git a/pkg/measurements/pod_latency.go b/pkg/measurements/pod_latency.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/cloud-bulldozer/kube-burner/log"
+	"github.com/cloud-bulldozer/kube-burner/pkg/config"
 	"github.com/cloud-bulldozer/kube-burner/pkg/measurements/metrics"
 	"github.com/cloud-bulldozer/kube-burner/pkg/measurements/types"
 	v1 "k8s.io/api/core/v1"
@@ -44,13 +45,14 @@ type podMetric struct {
 	containersReady        time.Time
 	ContainersReadyLatency int `json:"containersReadyLatency"`
 	podReady               time.Time
-	PodReadyLatency        int    `json:"podReadyLatency"`
-	MetricName             string `json:"metricName"`
-	JobName                string `json:"jobName"`
-	UUID                   string `json:"uuid"`
-	Namespace              string `json:"namespace"`
-	Name                   string `json:"podName"`
-	NodeName               string `json:"nodeName"`
+	PodReadyLatency        int        `json:"podReadyLatency"`
+	MetricName             string     `json:"metricName"`
+	JobName                string     `json:"jobName"`
+	JobConfig              config.Job `json:"jobConfig"`
+	UUID                   string     `json:"uuid"`
+	Namespace              string     `json:"namespace"`
+	Name                   string     `json:"podName"`
+	NodeName               string     `json:"nodeName"`
 }
 
 type podLatency struct {
@@ -68,6 +70,8 @@ func init() {
 func (p *podLatency) handleCreatePod(obj interface{}) {
 	now := time.Now().UTC()
 	pod := obj.(*v1.Pod)
+	jc := factory.jobConfig
+	jc.Objects = nil // metric doesn't need this data
 	if _, exists := p.metrics[string(pod.UID)]; !exists {
 		if strings.Contains(pod.Namespace, factory.jobConfig.Namespace) {
 			p.metrics[string(pod.UID)] = podMetric{
@@ -76,6 +80,7 @@ func (p *podLatency) handleCreatePod(obj interface{}) {
 				Name:       pod.Name,
 				MetricName: podLatencyMeasurement,
 				UUID:       factory.uuid,
+				JobConfig:  *jc,
 				JobName:    factory.jobConfig.Name,
 			}
 		}
@@ -208,6 +213,8 @@ func (p *podLatency) normalizeMetrics() {
 func (p *podLatency) calcQuantiles() {
 	quantiles := []float64{0.5, 0.95, 0.99}
 	quantileMap := map[v1.PodConditionType][]int{}
+	jc := factory.jobConfig
+	jc.Objects = nil
 	for _, normLatency := range p.normLatencies {
 		quantileMap[v1.PodScheduled] = append(quantileMap[v1.PodScheduled], normLatency.(podMetric).SchedulingLatency)
 		quantileMap[v1.ContainersReady] = append(quantileMap[v1.ContainersReady], normLatency.(podMetric).ContainersReadyLatency)
@@ -220,6 +227,7 @@ func (p *podLatency) calcQuantiles() {
 			UUID:         factory.uuid,
 			Timestamp:    time.Now().UTC(),
 			JobName:      factory.jobConfig.Name,
+			JobConfig:    *jc,
 			MetricName:   podLatencyQuantilesMeasurement,
 		}
 		sort.Ints(v)

diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go
@@ -175,6 +175,7 @@ func (p *Prometheus) ScrapeJobsMetrics(indexer *indexers.Indexer) error {
 
 func (p *Prometheus) parseVector(metricName, query string, value model.Value, metrics *[]interface{}) error {
 	var jobName string
+	var jobConfig config.Job
 	data, ok := value.(model.Vector)
 	if !ok {
 		return fmt.Errorf("unsupported result format: %s", value.Type().String())
@@ -183,6 +184,8 @@ func (p *Prometheus) parseVector(metricName, query string, value model.Value, me
 		for _, prometheusJob := range p.JobList {
 			if v.Timestamp.Time().Before(prometheusJob.End) {
 				jobName = prometheusJob.Name
+				jobConfig = prometheusJob.JobConfig
+				jobConfig.Objects = nil // no need to insert this into the metric.
 			}
 		}
 		m := metric{
@@ -191,6 +194,7 @@ func (p *Prometheus) parseVector(metricName, query string, value model.Value, me
 			Query:      query,
 			MetricName: metricName,
 			JobName:    jobName,
+			JobConfig:  jobConfig,
 		}
 		for k, v := range v.Metric {
 			if k == "__name__" {
@@ -211,6 +215,7 @@ func (p *Prometheus) parseVector(metricName, query string, value model.Value, me
 
 func (p *Prometheus) parseMatrix(metricName, query string, value model.Value, metrics *[]interface{}) error {
 	var jobName string
+	var jobConfig config.Job
 	data, ok := value.(model.Matrix)
 	if !ok {
 		return fmt.Errorf("unsupported result format: %s", value.Type().String())
@@ -220,6 +225,8 @@ func (p *Prometheus) parseMatrix(metricName, query string, value model.Value, me
 			for _, job := range p.JobList {
 				if val.Timestamp.Time().Before(job.End) {
 					jobName = job.Name
+					jobConfig = job.JobConfig
+					jobConfig.Objects = nil // no need to insert this into the metric.
 				}
 			}
 			m := metric{
@@ -228,6 +235,7 @@ func (p *Prometheus) parseMatrix(metricName, query string, value model.Value, me
 				Query:      query,
 				MetricName: metricName,
 				JobName:    jobName,
+				JobConfig:  jobConfig,
 				Timestamp:  val.Timestamp.Time(),
 			}
 			for k, v := range v.Metric {

diff --git a/pkg/prometheus/types.go b/pkg/prometheus/types.go
@@ -33,9 +33,10 @@ type Prometheus struct {
 }
 
 type Job struct {
-	Start time.Time
-	End   time.Time
-	Name  string
+	Start     time.Time
+	End       time.Time
+	Name      string
+	JobConfig config.Job
 }
 
 // This object implements RoundTripper
@@ -62,4 +63,5 @@ type metric struct {
 	Query      string            `json:"query"`
 	MetricName string            `json:"metricName,omitempty"`
 	JobName    string            `json:"jobName,omitempty"`
+	JobConfig  config.Job        `json:"jobConfig,omitempty"`
 }