From 7a3e6bb2eca84a03a7ae4c32c858fcec1461f639 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 24 Jan 2023 10:42:07 +0100 Subject: [PATCH] Add kubelet and cri-o metrics Signed-off-by: Raul Sevilla --- .../ocp-config/metrics-aggregated.yml | 16 +++++++++-- cmd/kube-burner/ocp-config/metrics.yml | 2 ++ pkg/workloads/helpers.go | 27 +++++++++---------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/cmd/kube-burner/ocp-config/metrics-aggregated.yml b/cmd/kube-burner/ocp-config/metrics-aggregated.yml index f0705b74e..d718bd6d0 100644 --- a/cmd/kube-burner/ocp-config/metrics-aggregated.yml +++ b/cmd/kube-burner/ocp-config/metrics-aggregated.yml @@ -72,9 +72,21 @@ - query: avg(node_memory_MemTotal_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)") metricName: nodeMemoryTotal-Infra + instant: true + +# Kubelet & CRI-O runtime metrics + +- query: irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100 and on (node) topk(3,avg_over_time(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"}) + metricName: kubeletCPU + +- query: process_resident_memory_bytes{service="kubelet",job="kubelet"} and on (node) topk(3,max_over_time(irate(process_resident_memory_bytes{service="kubelet",job="kubelet"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"}) + metricName: kubeletMemory + +- query: irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100 and on (node) topk(3,avg_over_time(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"}) + metricName: crioCPU -- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_pod"}[2m]) > 0 - metricName: containerNetworkSetupLatency +- query: process_resident_memory_bytes{service="kubelet",job="crio"} and on (node) topk(3,max_over_time(irate(process_resident_memory_bytes{service="kubelet",job="crio"}[2m])[{{ .elapsed }}:]) and on (node) kube_node_role{role="worker"}) + metricName: crioMemory # Etcd metrics diff --git a/cmd/kube-burner/ocp-config/metrics.yml b/cmd/kube-burner/ocp-config/metrics.yml index eff1d5c54..6fb6a8022 100644 --- a/cmd/kube-burner/ocp-config/metrics.yml +++ b/cmd/kube-burner/ocp-config/metrics.yml @@ -75,9 +75,11 @@ - query: node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)") metricName: nodeMemoryTotal-Workers + instant: true - query: node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)") metricName: nodeMemoryTotal-Infra + instant: true # Etcd metrics diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index 69e99ae8e..fd4429263 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -209,28 +209,27 @@ func (wh *WorkloadHelper) extractWorkload(workload, metrics string) error { if err != nil { return err } - for _, f := range dirContent { - fileContent, _ := wh.ocpConfig.ReadFile(path.Join(ocpCfgDir, workload, f.Name())) - fd, err := os.Create(f.Name()) + createFile := func(filePath, fileName string) error { + fileContent, _ := wh.ocpConfig.ReadFile(filePath) + fd, err := os.Create(fileName) if err != nil { return err } + defer fd.Close() fd.Write(fileContent) - fd.Close() + return nil } - fileContent, _ := wh.ocpConfig.ReadFile(path.Join(ocpCfgDir, metrics)) - fd, err := os.Create(metricsProfile) - if err != nil { + for _, f := range dirContent { + err := createFile(path.Join(ocpCfgDir, workload, f.Name()), f.Name()) + if err != nil { + return err + } + } + if err = createFile(path.Join(ocpCfgDir, metrics), metricsProfile); err != nil { return err } - fd.Write(fileContent) - fd.Close() - fileContent, _ = wh.ocpConfig.ReadFile(path.Join(ocpCfgDir, alertsProfile)) - fd, err = os.Create(alertsProfile) - if err != nil { + if err = createFile(path.Join(ocpCfgDir, alertsProfile), alertsProfile); err != nil { return err } - fd.Write(fileContent) - fd.Close() return nil }