Generate job summary document

Signed-off-by: Raul Sevilla <[email protected]>

README.md

@@ -14,6 +14,7 @@
   - [Job types](#job-types)
   - [Injected variables](#injected-variables)
   - [Metrics profile](#metrics-profile)
+  - [Job Summary](#job-summary)
   - [Indexers](#indexers)
   - [Measurements](#measurements)
     - [Pod latency](#pod-latency)
@@ -294,6 +295,50 @@ metrics:
     metricName: nodeCPU
 ```
 
+### Job Summary
+
+In case indexing is enabled, at the end of each job, a document holding the job summary is indexed. This is useful to indentify the options the job was parameters with:
+
+This document looks like:
+
+```json
+{
+  "timestamp": "2020-11-13T13:55:31.654185032+01:00",
+  "uuid": "bdb7584a-d2cd-4185-8bfa-1387cc31f99e",
+  "metricName": "jobSummary",
+  "elapsedTime": 8.768932955,
+  "jobConfig": {
+    "jobIterations": 10,
+    "jobIterationDelay": 0,
+    "jobPause": 0,
+    "name": "kubelet-density",
+    "objects": [
+      {
+        "objectTemplate": "templates/pod.yml",
+        "replicas": 1,
+        "inputVars": {
+          "containerImage": "gcr.io/google_containers/pause-amd64:3.0"
+        }
+      }
+    ],
+    "jobType": "create",
+    "qps": 5,
+    "burst": 5,
+    "namespace": "kubelet-density",
+    "waitFor": null,
+    "maxWaitTimeout": 43200000000000,
+    "waitForDeletion": true,
+    "podWait": false,
+    "waitWhenFinished": true,
+    "cleanup": true,
+    "namespaced": false,
+    "namespacedIterations": false,
+    "verifyObjects": true,
+    "errorOnVerify": false
+  }
+}
+```
+
 ### Indexers
 
 `kube-burner` is able to **index the collected prometheus metrics** into a given Indexer. 

cmd/kube-burner.go

@@ -234,7 +234,11 @@ func steps(uuid string, p *prometheus.Prometheus, prometheusStep time.Duration)
 		case config.DeletionJob:
 			job.RunDeleteJob()
 		}
-		log.Infof("Job %s took %.2f seconds", job.Config.Name, job.End.Sub(job.Start).Seconds())
+		elapsedTime := job.End.Sub(job.Start).Seconds()
+		log.Infof("Job %s took %.2f seconds", job.Config.Name, elapsedTime)
+		if config.ConfigSpec.GlobalConfig.IndexerConfig.Enabled {
+			burner.IndexMetadataInfo(indexer, uuid, elapsedTime, job.Config)
+		}
 		if job.Config.JobPause > 0 {
 			log.Infof("Pausing for %v before next job", job.Config.JobPause)
 			time.Sleep(job.Config.JobPause)

pkg/burner/metadata.go

@@ -0,0 +1,46 @@
+// Copyright 2020 The Kube-burner Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package burner
+
+import (
+	"time"
+
+	"github.com/cloud-bulldozer/kube-burner/log"
+	"github.com/cloud-bulldozer/kube-burner/pkg/config"
+	"github.com/cloud-bulldozer/kube-burner/pkg/indexers"
+)
+
+type metadata struct {
+	Timestamp   time.Time  `json:"timestamp"`
+	UUID        string     `json:"uuid"`
+	MetricName  string     `json:"metricName"`
+	ElapsedTime float64    `json:"elapsedTime"`
+	JobConfig   config.Job `json:"jobConfig"`
+}
+
+// IndexMetadataInfo Generates and indexes a document with metadata information of the passed job
+func IndexMetadataInfo(indexer *indexers.Indexer, uuid string, elapsedTime float64, jobConfig config.Job) {
+	metadataInfo := []interface{}{
+		metadata{
+			UUID:        uuid,
+			ElapsedTime: elapsedTime,
+			JobConfig:   jobConfig,
+			MetricName:  "jobSummary",
+			Timestamp:   time.Now(),
+		},
+	}
+	log.Infof("Indexing metadata information: %+v", metadataInfo)
+	(*indexer).Index(config.ConfigSpec.GlobalConfig.IndexerConfig.DefaultIndex, metadataInfo)
+}

pkg/config/types.go

@@ -96,58 +96,58 @@ type GlobalConfig struct {
 // Object defines an object that kube-burner will create
 type Object struct {
 	// ObjectTemplate path to a valid YAML definition of a k8s resource
-	ObjectTemplate string `yaml:"objectTemplate"`
+	ObjectTemplate string `yaml:"objectTemplate" json:"objectTemplate,omitempty"`
 	// Replicas number of replicas to create of the given object
-	Replicas int `yaml:"replicas"`
+	Replicas int `yaml:"replicas" json:"replicas,omitempty"`
 	// InputVars contains a map of arbitrary input variables
-	// that can be introduced by the users
-	InputVars map[string]string `yaml:"inputVars"`
+	// that can be introduced by users
+	InputVars map[string]string `yaml:"inputVars" json:"inputVars,omitempty"`
 	// Kind object kind to delete
-	Kind string `yaml:"kind"`
+	Kind string `yaml:"kind" json:"kind,omitempty"`
 	// APIVersion apiVersion of the object to remove
-	APIVersion string `yaml:"apiVersion"`
+	APIVersion string `yaml:"apiVersion" json:"apiVersion,omitempty"`
 	// LabelSelector objects with this labels will be removed
-	LabelSelector map[string]string `yaml:"labelSelector"`
+	LabelSelector map[string]string `yaml:"labelSelector" json:"labelSelector,omitempty"`
 }
 
 // Job defines a kube-burner job
 type Job struct {
 	// IterationCount how many times to execute the job
-	JobIterations int `yaml:"jobIterations"`
+	JobIterations int `yaml:"jobIterations" json:"jobIterations"`
 	// IterationDelay how much time to wait between each job iteration
-	JobIterationDelay time.Duration `yaml:"jobIterationDelay"`
+	JobIterationDelay time.Duration `yaml:"jobIterationDelay" json:"jobIterationDelay"`
 	// JobPause how much time to pause after finishing the job
-	JobPause time.Duration `yaml:"jobPause"`
+	JobPause time.Duration `yaml:"jobPause" json:"jobPause"`
 	// Name job name
-	Name string `yaml:"name"`
+	Name string `yaml:"name" json:"name"`
 	// Objects list of objects
-	Objects []Object `yaml:"objects"`
+	Objects []Object `yaml:"objects" json:"objects"`
 	// JobType type of job
-	JobType JobType `yaml:"jobType"`
+	JobType JobType `yaml:"jobType" json:"jobType"`
 	// Max number of queries per second
-	QPS int `yaml:"qps"`
+	QPS int `yaml:"qps" json:"qps"`
 	// Maximum burst for throttle
-	Burst int `yaml:"burst"`
+	Burst int `yaml:"burst" json:"burst"`
 	// Namespace namespace base name to use
-	Namespace string `yaml:"namespace"`
+	Namespace string `yaml:"namespace" json:"namespace"`
 	// WaitFor list of objects to wait for, if not specified wait for all
-	WaitFor []string `yaml:"waitFor"`
+	WaitFor []string `yaml:"waitFor" json:"waitFor"`
 	// MaxWaitTimeout maximum wait period
-	MaxWaitTimeout time.Duration `yaml:"maxWaitTimeout"`
+	MaxWaitTimeout time.Duration `yaml:"maxWaitTimeout" json:"maxWaitTimeout"`
 	// WaitForDeletion wait for objects to be definitively deleted
-	WaitForDeletion bool `yaml:"waitForDeletion"`
+	WaitForDeletion bool `yaml:"waitForDeletion" json:"waitForDeletion"`
 	// PodWait wait for all pods to be running before moving forward to the next iteration
-	PodWait bool `yaml:"podWait"`
+	PodWait bool `yaml:"podWait" json:"podWait"`
 	// WaitWhenFinished Wait for pods to be running when all job iterations are completed
-	WaitWhenFinished bool `yaml:"waitWhenFinished"`
+	WaitWhenFinished bool `yaml:"waitWhenFinished" json:"waitWhenFinished"`
 	// Cleanup clean up old namespaces
-	Cleanup bool `yaml:"cleanup"`
+	Cleanup bool `yaml:"cleanup" json:"cleanup"`
 	// whether to create namespaces or not with each job iteration
-	Namespaced bool `yaml:"namespaced"`
+	Namespaced bool `yaml:"namespaced" json:"namespaced"`
 	// NamespacedIterations create a namespace per job iteration
-	NamespacedIterations bool `yaml:"namespacedIterations"`
+	NamespacedIterations bool `yaml:"namespacedIterations" json:"namespacedIterations"`
 	// VerifyObjects verify object count after running the job
-	VerifyObjects bool `yaml:"verifyObjects"`
+	VerifyObjects bool `yaml:"verifyObjects" json:"verifyObjects"`
 	// ErrorOnVerify exit when verification fails
-	ErrorOnVerify bool `yaml:"errorOnVerify"`
+	ErrorOnVerify bool `yaml:"errorOnVerify" json:"errorOnVerify"`
 }