Add kubelet and cri-o metrics

Signed-off-by: Raul Sevilla <[email protected]>
rsevilla87 · Jan 24, 2023 · 7a3e6bb · 7a3e6bb
1 parent 7ebc1ef
commit 7a3e6bb
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 16 deletions.
diff --git a/cmd/kube-burner/ocp-config/metrics-aggregated.yml b/cmd/kube-burner/ocp-config/metrics-aggregated.yml
@@ -72,9 +72,21 @@
 
 - query: avg(node_memory_MemTotal_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")
   metricName: nodeMemoryTotal-Infra
+  instant: true
+
+# Kubelet & CRI-O runtime metrics
+
+- query: irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100 and on (node) topk(3,avg_over_time(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"})
+  metricName: kubeletCPU
+
+- query: process_resident_memory_bytes{service="kubelet",job="kubelet"} and on (node) topk(3,max_over_time(irate(process_resident_memory_bytes{service="kubelet",job="kubelet"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"})
+  metricName: kubeletMemory
+
+- query: irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100 and on (node) topk(3,avg_over_time(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"})
+  metricName: crioCPU
 
-- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_pod"}[2m]) > 0
-  metricName: containerNetworkSetupLatency
+- query: process_resident_memory_bytes{service="kubelet",job="crio"} and on (node) topk(3,max_over_time(irate(process_resident_memory_bytes{service="kubelet",job="crio"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"})
+  metricName: crioMemory
 
 # Etcd metrics
 

diff --git a/cmd/kube-burner/ocp-config/metrics.yml b/cmd/kube-burner/ocp-config/metrics.yml
@@ -75,9 +75,11 @@
 
 - query: node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")
   metricName: nodeMemoryTotal-Workers
+  instant: true
 
 - query: node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")
   metricName: nodeMemoryTotal-Infra
+  instant: true
 
 # Etcd metrics
 

diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go
@@ -209,28 +209,27 @@ func (wh *WorkloadHelper) extractWorkload(workload, metrics string) error {
 	if err != nil {
 		return err
 	}
-	for _, f := range dirContent {
-		fileContent, _ := wh.ocpConfig.ReadFile(path.Join(ocpCfgDir, workload, f.Name()))
-		fd, err := os.Create(f.Name())
+	createFile := func(filePath, fileName string) error {
+		fileContent, _ := wh.ocpConfig.ReadFile(filePath)
+		fd, err := os.Create(fileName)
 		if err != nil {
 			return err
 		}
+		defer fd.Close()
 		fd.Write(fileContent)
-		fd.Close()
+		return nil
 	}
-	fileContent, _ := wh.ocpConfig.ReadFile(path.Join(ocpCfgDir, metrics))
-	fd, err := os.Create(metricsProfile)
-	if err != nil {
+	for _, f := range dirContent {
+		err := createFile(path.Join(ocpCfgDir, workload, f.Name()), f.Name())
+		if err != nil {
+			return err
+		}
+	}
+	if err = createFile(path.Join(ocpCfgDir, metrics), metricsProfile); err != nil {
 		return err
 	}
-	fd.Write(fileContent)
-	fd.Close()
-	fileContent, _ = wh.ocpConfig.ReadFile(path.Join(ocpCfgDir, alertsProfile))
-	fd, err = os.Create(alertsProfile)
-	if err != nil {
+	if err = createFile(path.Join(ocpCfgDir, alertsProfile), alertsProfile); err != nil {
 		return err
 	}
-	fd.Write(fileContent)
-	fd.Close()
 	return nil
 }