Emit 8 new pod_status_* metrics for containerinsights (open-telemetry#25

)

Emit 8 new pod_status_* metrics for containerinsights (open-telemetry#25)

internal/aws/containerinsight/const.go

@@ -112,6 +112,13 @@ const (
 	StatusTerminated                  = "status_terminated"
 	StatusWaiting                     = "status_waiting"
 	StatusWaitingReasonCrashed        = "status_waiting_reason_crashed"
+	StatusPending                     = "status_pending"
+	StatusSucceeded                   = "status_succeeded"
+	StatusFailed                      = "status_failed"
+	StatusUnknown                     = "status_unknown"
+	StatusReady                       = "status_ready"
+	StatusScheduled                   = "status_scheduled"
+	StatusInitialized                 = "status_initialized"
 
 	RunningPodCount       = "number_of_running_pods"
 	RunningContainerCount = "number_of_running_containers"

receiver/awscontainerinsightreceiver/internal/stores/podstore.go

@@ -47,6 +47,20 @@ const (
 
 var (
 	re = regexp.MustCompile(splitRegexStr)
+
+	PodPhaseMetricNames = map[corev1.PodPhase]string{
+		corev1.PodPending:   ci.MetricName(ci.TypePod, ci.StatusPending),
+		corev1.PodRunning:   ci.MetricName(ci.TypePod, ci.StatusRunning),
+		corev1.PodSucceeded: ci.MetricName(ci.TypePod, ci.StatusSucceeded),
+		corev1.PodFailed:    ci.MetricName(ci.TypePod, ci.StatusFailed),
+		corev1.PodUnknown:   ci.MetricName(ci.TypePod, ci.StatusUnknown),
+	}
+
+	PodConditionMetricNames = map[corev1.PodConditionType]string{
+		corev1.PodReady:       ci.MetricName(ci.TypePod, ci.StatusReady),
+		corev1.PodScheduled:   ci.MetricName(ci.TypePod, ci.StatusScheduled),
+		corev1.PodInitialized: ci.MetricName(ci.TypePod, ci.StatusInitialized),
+	}
 )
 
 type cachedEntry struct {
@@ -453,6 +467,9 @@ func (p *PodStore) decorateMem(metric CIMetric, pod *corev1.Pod) {
 func (p *PodStore) addStatus(metric CIMetric, pod *corev1.Pod) {
 	if metric.GetTag(ci.MetricType) == ci.TypePod {
 		metric.AddTag(ci.PodStatus, string(pod.Status.Phase))
+		p.addPodStatusMetrics(metric, pod)
+		p.addPodConditionMetrics(metric, pod)
+
 		var curContainerRestarts int
 		for _, containerStatus := range pod.Status.ContainerStatuses {
 			curContainerRestarts += int(containerStatus.RestartCount)
@@ -528,6 +545,35 @@ func (p *PodStore) addStatus(metric CIMetric, pod *corev1.Pod) {
 	}
 }
 
+func (p *PodStore) addPodStatusMetrics(metric CIMetric, pod *corev1.Pod) {
+	for _, metricName := range PodPhaseMetricNames {
+		metric.AddField(metricName, 0)
+	}
+
+	statusMetricName, validStatus := PodPhaseMetricNames[pod.Status.Phase]
+	if validStatus {
+		metric.AddField(statusMetricName, 1)
+	}
+}
+
+func (p *PodStore) addPodConditionMetrics(metric CIMetric, pod *corev1.Pod) {
+	for _, metricName := range PodConditionMetricNames {
+		metric.AddField(metricName, 0)
+	}
+
+	for _, condition := range pod.Status.Conditions {
+		if condition.Status != corev1.ConditionTrue {
+			continue
+		}
+
+		conditionKey := condition.Type
+		statusMetricName, conditionKeyValid := PodConditionMetricNames[conditionKey]
+		if conditionKeyValid {
+			metric.AddField(statusMetricName, 1)
+		}
+	}
+}
+
 // It could be used to get limit/request(depend on the passed-in fn) per pod
 // return the sum of ResourceSetting and a bool which indicate whether all container set Resource
 func getResourceSettingForPod(pod *corev1.Pod, bound uint64, resource corev1.ResourceName, fn func(resource corev1.ResourceName, spec corev1.Container) (uint64, bool)) (uint64, bool) {

receiver/awscontainerinsightreceiver/internal/stores/podstore_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"testing"
 	"time"
 
@@ -293,6 +294,90 @@ func TestPodStore_addContainerCount(t *testing.T) {
 	assert.Equal(t, int(1), metric.GetField(ci.MetricName(ci.TypePod, ci.ContainerCount)).(int))
 }
 
+const (
+	PodFailedMetricName      = "pod_status_failed"
+	PodPendingMetricName     = "pod_status_pending"
+	PodRunningMetricName     = "pod_status_running"
+	PodSucceededMetricName   = "pod_status_succeeded"
+	PodUnknownMetricName     = "pod_status_unknown"
+	PodReadyMetricName       = "pod_status_ready"
+	PodScheduledMetricName   = "pod_status_scheduled"
+	PodInitializedMetricName = "pod_status_initialized"
+)
+
+func TestPodStore_addStatus_adds_pod_failed_metric(t *testing.T) {
+	decoratedResultMetric := runAddStatusToGetDecoratedCIMetric(
+		"./test_resources/pod_in_phase_failed.json")
+
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodFailedMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName))
+}
+
+func TestPodStore_addStatus_adds_pod_pending_metric(t *testing.T) {
+	decoratedResultMetric := runAddStatusToGetDecoratedCIMetric(
+		"./test_resources/pod_in_phase_pending.json")
+
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodFailedMetricName))
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodPendingMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName))
+}
+
+func TestPodStore_addStatus_adds_pod_running_metric(t *testing.T) {
+	decoratedResultMetric := runAddStatusToGetDecoratedCIMetric(
+		"./test_resources/pod_in_phase_running.json")
+
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodFailedMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName))
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodRunningMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName))
+}
+
+func TestPodStore_addStatus_adds_pod_succeeded_metric(t *testing.T) {
+	decoratedResultMetric := runAddStatusToGetDecoratedCIMetric(
+		"./test_resources/pod_in_phase_succeeded.json")
+
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodFailedMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName))
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodSucceededMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName))
+}
+
+func TestPodStore_addStatus_adds_pod_unknown_metric(t *testing.T) {
+	decoratedResultMetric := runAddStatusToGetDecoratedCIMetric(
+		"./test_resources/pod_in_phase_unknown.json")
+
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodFailedMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName))
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodUnknownMetricName))
+}
+
+func TestPodStore_addStatus_adds_all_pod_conditions_as_metrics_when_true_false_unknown(t *testing.T) {
+	decoratedResultMetric := runAddStatusToGetDecoratedCIMetric(
+		"./test_resources/all_pod_conditions_valid.json")
+
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodInitializedMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodReadyMetricName))
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodScheduledMetricName))
+}
+
+func TestPodStore_addStatus_adds_all_pod_conditions_as_metrics_when_unexpected(t *testing.T) {
+	decoratedResultMetric := runAddStatusToGetDecoratedCIMetric(
+		"./test_resources/one_pod_condition_invalid.json")
+
+	assert.Equal(t, 0, decoratedResultMetric.GetField(PodInitializedMetricName))
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodReadyMetricName))
+	assert.Equal(t, 1, decoratedResultMetric.GetField(PodScheduledMetricName))
+}
+
 func TestPodStore_addStatus(t *testing.T) {
 	pod := getBaseTestPodInfo()
 	tags := map[string]string{ci.MetricType: ci.TypePod, ci.K8sNamespace: "default", ci.K8sPodNameKey: "cpu-limit"}
@@ -716,3 +801,30 @@ func TestPodStore_Decorate(t *testing.T) {
 	ok = podStore.Decorate(ctx, metric, kubernetesBlob)
 	assert.False(t, ok)
 }
+
+func runAddStatusToGetDecoratedCIMetric(podInfoSourceFileName string) CIMetric {
+	podInfo := generatePodInfo(podInfoSourceFileName)
+	podStore := getPodStore()
+	rawCIMetric := generateRawCIMetric()
+	podStore.addStatus(rawCIMetric, podInfo)
+	return rawCIMetric
+}
+
+func generatePodInfo(sourceFileName string) *corev1.Pod {
+	podInfoJSON, err := os.ReadFile(sourceFileName)
+	if err != nil {
+		panic(fmt.Sprintf("reading file failed %v", err))
+	}
+	pods := corev1.PodList{}
+	err = json.Unmarshal(podInfoJSON, &pods)
+	if err != nil {
+		panic(fmt.Sprintf("unmarshal pod err %v", err))
+	}
+	return &pods.Items[0]
+}
+
+func generateRawCIMetric() CIMetric {
+	tags := map[string]string{ci.MetricType: ci.TypePod, ci.K8sNamespace: "default", ci.K8sPodNameKey: "cpu-limit"}
+	fields := map[string]interface{}{ci.MetricName(ci.TypePod, ci.CPUTotal): float64(1)}
+	return generateMetric(fields, tags)
+}

receiver/awscontainerinsightreceiver/internal/stores/test_resources/all_pod_conditions_valid.json

@@ -0,0 +1,156 @@
+{
+  "kind": "PodList",
+  "apiVersion": "v1",
+  "metadata": {
+
+  },
+  "items": [
+    {
+      "metadata": {
+        "name": "cpu-limit",
+        "namespace": "default",
+        "ownerReferences": [
+          {
+            "apiVersion": "apps/v1",
+            "blockOwnerDeletion": true,
+            "controller": true,
+            "kind": "DaemonSet",
+            "name": "DaemonSetTest",
+            "uid": "36779a62-4aca-11e9-977b-0672b6c6fc94"
+          }
+        ],
+        "selfLink": "/api/v1/namespaces/default/pods/cpu-limit",
+        "uid": "764d01e1-2a2f-11e9-95ea-0a695d7ce286",
+        "resourceVersion": "5671573",
+        "creationTimestamp": "2019-02-06T16:51:34Z",
+        "labels": {
+          "app": "hello_test"
+        },
+        "annotations": {
+          "kubernetes.io/config.seen": "2019-02-19T00:06:56.109155665Z",
+          "kubernetes.io/config.source": "api"
+        }
+      },
+      "spec": {
+        "volumes": [
+          {
+            "name": "default-token-tlgw7",
+            "secret": {
+              "secretName": "default-token-tlgw7",
+              "defaultMode": 420
+            }
+          }
+        ],
+        "containers": [
+          {
+            "name": "ubuntu",
+            "image": "ubuntu",
+            "command": [
+              "/bin/bash"
+            ],
+            "args": [
+              "-c",
+              "sleep 300000000"
+            ],
+            "resources": {
+              "limits": {
+                "cpu": "10m",
+                "memory": "50Mi"
+              },
+              "requests": {
+                "cpu": "10m",
+                "memory": "50Mi"
+              }
+            },
+            "volumeMounts": [
+              {
+                "name": "default-token-tlgw7",
+                "readOnly": true,
+                "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+              }
+            ],
+            "terminationMessagePath": "/dev/termination-log",
+            "terminationMessagePolicy": "File",
+            "imagePullPolicy": "Always"
+          }
+        ],
+        "restartPolicy": "Always",
+        "terminationGracePeriodSeconds": 30,
+        "dnsPolicy": "ClusterFirst",
+        "serviceAccountName": "default",
+        "serviceAccount": "default",
+        "nodeName": "ip-192-168-67-127.us-west-2.compute.internal",
+        "securityContext": {
+
+        },
+        "schedulerName": "default-scheduler",
+        "tolerations": [
+          {
+            "key": "node.kubernetes.io/not-ready",
+            "operator": "Exists",
+            "effect": "NoExecute",
+            "tolerationSeconds": 300
+          },
+          {
+            "key": "node.kubernetes.io/unreachable",
+            "operator": "Exists",
+            "effect": "NoExecute",
+            "tolerationSeconds": 300
+          }
+        ],
+        "priority": 0
+      },
+      "status": {
+        "phase": "Running",
+        "conditions": [
+          {
+            "type": "Initialized",
+            "status": "True",
+            "lastProbeTime": null,
+            "lastTransitionTime": "2019-02-06T16:51:34Z"
+          },
+          {
+            "type": "Ready",
+            "status": "False",
+            "lastProbeTime": null,
+            "lastTransitionTime": "2019-02-06T16:51:43Z"
+          },
+          {
+            "type": "ContainersReady",
+            "status": "True",
+            "lastProbeTime": null,
+            "lastTransitionTime": null
+          },
+          {
+            "type": "PodScheduled",
+            "status": "Unknown",
+            "lastProbeTime": null,
+            "lastTransitionTime": "2019-02-06T16:51:34Z"
+          }
+        ],
+        "hostIP": "192.168.67.127",
+        "podIP": "192.168.76.93",
+        "startTime": "2019-02-06T16:51:34Z",
+        "containerStatuses": [
+          {
+            "name": "ubuntu",
+            "state": {
+              "running": {
+                "startedAt": "2019-02-06T16:51:42Z"
+              }
+            },
+            "lastState": {
+
+            },
+            "ready": true,
+            "restartCount": 0,
+            "image": "ubuntu:latest",
+            "imageID": "docker-pullable://ubuntu@sha256:7a47ccc3bbe8a451b500d2b53104868b46d60ee8f5b35a24b41a86077c650210",
+            "containerID": "docker://637631e2634ea92c0c1aa5d24734cfe794f09c57933026592c12acafbaf6972c"
+          }
+        ],
+        "qosClass": "Guaranteed"
+      }
+    }
+  ]
+}