From 0a196016738e8302e25d2cd9ffc2b4eecd8a4c07 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 16 Jun 2020 15:00:39 +0100 Subject: [PATCH 1/4] feat: add additional tags/fields to pod metrics --- plugins/inputs/kube_inventory/README.md | 9 +- plugins/inputs/kube_inventory/pod.go | 14 ++- plugins/inputs/kube_inventory/pod_test.go | 110 +++++++++++++++++++++- 3 files changed, 124 insertions(+), 9 deletions(-) diff --git a/plugins/inputs/kube_inventory/README.md b/plugins/inputs/kube_inventory/README.md index f017b18c6800b..33b05d800ec6c 100644 --- a/plugins/inputs/kube_inventory/README.md +++ b/plugins/inputs/kube_inventory/README.md @@ -206,10 +206,12 @@ subjects: - namespace - node_name - pod_name + - state + - ready - fields: - restarts_total - - state - - terminated_reason + - state_code + - state_reason - resource_requests_cpu_units - resource_requests_memory_bytes - resource_limits_cpu_units @@ -267,7 +269,6 @@ The persistentvolumeclaim "phase" is saved in the `phase` tag with a correlated | pending | 2 | | unknown | 3 | - ### Example Output: ``` @@ -278,7 +279,7 @@ kubernetes_node,node_name=ip-172-17-0-2.internal allocatable_pods=110i,capacity_ kubernetes_persistentvolume,phase=Released,pv_name=pvc-aaaaaaaa-bbbb-cccc-1111-222222222222,storageclass=ebs-1-retain phase_type=3i 1547597616000000000 kubernetes_persistentvolumeclaim,namespace=default,phase=Bound,pvc_name=data-etcd-0,storageclass=ebs-1-retain phase_type=0i 1547597615000000000 kubernetes_pod,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1 last_transition_time=1547578322000000000i,ready="false" 1547597616000000000 -kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,terminated_reason="",resource_requests_memory_bytes=524288000 1547597616000000000 +kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running,ready=ready resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,state_reason="",terminated_reason="",resource_requests_memory_bytes=524288000 1547597616000000000 kubernetes_statefulset,namespace=default,statefulset_name=etcd replicas_updated=3i,spec_replicas=3i,observed_generation=1i,created=1544101669000000000i,generation=1i,replicas=3i,replicas_current=3i,replicas_ready=3i 1547597616000000000 ``` diff --git a/plugins/inputs/kube_inventory/pod.go b/plugins/inputs/kube_inventory/pod.go index 7b5207616d412..8abdb012638df 100644 --- a/plugins/inputs/kube_inventory/pod.go +++ b/plugins/inputs/kube_inventory/pod.go @@ -3,7 +3,7 @@ package kube_inventory import ( "context" - "github.com/ericchiang/k8s/apis/core/v1" + v1 "github.com/ericchiang/k8s/apis/core/v1" "github.com/influxdata/telegraf" ) @@ -37,7 +37,9 @@ func (ki *KubernetesInventory) gatherPod(p v1.Pod, acc telegraf.Accumulator) err func gatherPodContainer(nodeName string, p v1.Pod, cs v1.ContainerStatus, c v1.Container, acc telegraf.Accumulator) { stateCode := 3 + stateReason := "" state := "unknown" + switch { case cs.State.Running != nil: stateCode = 0 @@ -45,22 +47,32 @@ func gatherPodContainer(nodeName string, p v1.Pod, cs v1.ContainerStatus, c v1.C case cs.State.Terminated != nil: stateCode = 1 state = "terminated" + stateReason = cs.State.Terminated.GetReason() case cs.State.Waiting != nil: stateCode = 2 state = "waiting" + stateReason = cs.State.Waiting.GetReason() + } + + ready := "unready" + if cs.GetReady() { + ready = "ready" } fields := map[string]interface{}{ "restarts_total": cs.GetRestartCount(), "state_code": stateCode, + "state_reason": stateReason, "terminated_reason": cs.State.Terminated.GetReason(), } + tags := map[string]string{ "container_name": *c.Name, "namespace": *p.Metadata.Namespace, "node_name": *p.Spec.NodeName, "pod_name": *p.Metadata.Name, "state": state, + "ready": ready, } req := c.Resources.Requests diff --git a/plugins/inputs/kube_inventory/pod_test.go b/plugins/inputs/kube_inventory/pod_test.go index 50b093880dbf1..877cd5a045b76 100644 --- a/plugins/inputs/kube_inventory/pod_test.go +++ b/plugins/inputs/kube_inventory/pod_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/ericchiang/k8s/apis/core/v1" + v1 "github.com/ericchiang/k8s/apis/core/v1" metav1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/ericchiang/k8s/apis/resource" "github.com/influxdata/telegraf/testutil" @@ -44,7 +44,43 @@ func TestPod(t *testing.T) { NodeName: toStrPtr("node1"), Containers: []*v1.Container{ { - Name: toStrPtr("forwarder"), + Name: toStrPtr("running"), + Image: toStrPtr("image1"), + Ports: []*v1.ContainerPort{ + { + ContainerPort: toInt32Ptr(8080), + Protocol: toStrPtr("TCP"), + }, + }, + Resources: &v1.ResourceRequirements{ + Limits: map[string]*resource.Quantity{ + "cpu": {String_: toStrPtr("100m")}, + }, + Requests: map[string]*resource.Quantity{ + "cpu": {String_: toStrPtr("100m")}, + }, + }, + }, + { + Name: toStrPtr("completed"), + Image: toStrPtr("image1"), + Ports: []*v1.ContainerPort{ + { + ContainerPort: toInt32Ptr(8080), + Protocol: toStrPtr("TCP"), + }, + }, + Resources: &v1.ResourceRequirements{ + Limits: map[string]*resource.Quantity{ + "cpu": {String_: toStrPtr("100m")}, + }, + Requests: map[string]*resource.Quantity{ + "cpu": {String_: toStrPtr("100m")}, + }, + }, + }, + { + Name: toStrPtr("waiting"), Image: toStrPtr("image1"), Ports: []*v1.ContainerPort{ { @@ -101,7 +137,7 @@ func TestPod(t *testing.T) { }, ContainerStatuses: []*v1.ContainerStatus{ { - Name: toStrPtr("forwarder"), + Name: toStrPtr("running"), State: &v1.ContainerState{ Running: &v1.ContainerStateRunning{ StartedAt: &metav1.Time{Seconds: toInt64Ptr(cond2.Unix())}, @@ -113,6 +149,34 @@ func TestPod(t *testing.T) { ImageID: toStrPtr("image_id1"), ContainerID: toStrPtr("docker://54abe32d0094479d3d"), }, + { + Name: toStrPtr("completed"), + State: &v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + StartedAt: &metav1.Time{Seconds: toInt64Ptr(cond2.Unix())}, + ExitCode: toInt32Ptr(0), + Reason: toStrPtr("Completed"), + }, + }, + Ready: toBoolPtr(false), + RestartCount: toInt32Ptr(3), + Image: toStrPtr("image1"), + ImageID: toStrPtr("image_id1"), + ContainerID: toStrPtr("docker://54abe32d0094479d3d"), + }, + { + Name: toStrPtr("waiting"), + State: &v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: toStrPtr("PodUninitialized"), + }, + }, + Ready: toBoolPtr(false), + RestartCount: toInt32Ptr(3), + Image: toStrPtr("image1"), + ImageID: toStrPtr("image_id1"), + ContainerID: toStrPtr("docker://54abe32d0094479d3d"), + }, }, }, Metadata: &metav1.ObjectMeta{ @@ -145,15 +209,53 @@ func TestPod(t *testing.T) { Fields: map[string]interface{}{ "restarts_total": int32(3), "state_code": 0, + "state_reason": "", "resource_requests_millicpu_units": int64(100), "resource_limits_millicpu_units": int64(100), }, Tags: map[string]string{ "namespace": "ns1", - "container_name": "forwarder", + "container_name": "running", "node_name": "node1", "pod_name": "pod1", "state": "running", + "ready": "ready", + }, + }, + { + Measurement: podContainerMeasurement, + Fields: map[string]interface{}{ + "restarts_total": int32(3), + "state_code": 1, + "state_reason": "Completed", + "resource_requests_millicpu_units": int64(100), + "resource_limits_millicpu_units": int64(100), + }, + Tags: map[string]string{ + "namespace": "ns1", + "container_name": "completed", + "node_name": "node1", + "pod_name": "pod1", + "state": "terminated", + "ready": "unready", + }, + }, + { + Measurement: podContainerMeasurement, + Fields: map[string]interface{}{ + "restarts_total": int32(3), + "state_code": 2, + "state_reason": "PodUninitialized", + "resource_requests_millicpu_units": int64(100), + "resource_limits_millicpu_units": int64(100), + }, + Tags: map[string]string{ + "namespace": "ns1", + "container_name": "waiting", + "node_name": "node1", + "pod_name": "pod1", + "state": "waiting", + "ready": "unready", }, }, }, From 41302b1583d39dcf6ee106d6ed0ac46d04bb2213 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 16 Jun 2020 18:42:06 +0100 Subject: [PATCH 2/4] chore: handle metric deprication of terminated_reason --- plugins/inputs/kube_inventory/README.md | 4 ++++ plugins/inputs/kube_inventory/pod.go | 11 +++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/kube_inventory/README.md b/plugins/inputs/kube_inventory/README.md index 33b05d800ec6c..90982472c399e 100644 --- a/plugins/inputs/kube_inventory/README.md +++ b/plugins/inputs/kube_inventory/README.md @@ -66,6 +66,9 @@ avoid cardinality issues: # tls_key = "/path/to/keyfile" ## Use TLS but skip chain & host verification # insecure_skip_verify = false + + ## Uncomment to remove deprecated metrics. + # fielddrop = ["terminated_reason"] ``` #### Kubernetes Permissions @@ -212,6 +215,7 @@ subjects: - restarts_total - state_code - state_reason + - terminated_reason (string, deprecated in 1.15: use `state_reason` instead) - resource_requests_cpu_units - resource_requests_memory_bytes - resource_limits_cpu_units diff --git a/plugins/inputs/kube_inventory/pod.go b/plugins/inputs/kube_inventory/pod.go index 8abdb012638df..f6947c601368b 100644 --- a/plugins/inputs/kube_inventory/pod.go +++ b/plugins/inputs/kube_inventory/pod.go @@ -54,25 +54,28 @@ func gatherPodContainer(nodeName string, p v1.Pod, cs v1.ContainerStatus, c v1.C stateReason = cs.State.Waiting.GetReason() } - ready := "unready" + readiness := "unready" if cs.GetReady() { - ready = "ready" + readiness = "ready" } fields := map[string]interface{}{ "restarts_total": cs.GetRestartCount(), "state_code": stateCode, - "state_reason": stateReason, "terminated_reason": cs.State.Terminated.GetReason(), } + if stateReason != "" { + fields["state_reason"] = stateReason + } + tags := map[string]string{ "container_name": *c.Name, "namespace": *p.Metadata.Namespace, "node_name": *p.Spec.NodeName, "pod_name": *p.Metadata.Name, "state": state, - "ready": ready, + "readiness": readiness, } req := c.Resources.Requests From fb4e2646e89fe29ff7714afec6e05421e9101af4 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 16 Jun 2020 18:43:37 +0100 Subject: [PATCH 3/4] chore: fix readiness tag name in docs --- plugins/inputs/kube_inventory/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/kube_inventory/README.md b/plugins/inputs/kube_inventory/README.md index 90982472c399e..013a8f3daf7bc 100644 --- a/plugins/inputs/kube_inventory/README.md +++ b/plugins/inputs/kube_inventory/README.md @@ -210,7 +210,7 @@ subjects: - node_name - pod_name - state - - ready + - readiness - fields: - restarts_total - state_code @@ -283,7 +283,7 @@ kubernetes_node,node_name=ip-172-17-0-2.internal allocatable_pods=110i,capacity_ kubernetes_persistentvolume,phase=Released,pv_name=pvc-aaaaaaaa-bbbb-cccc-1111-222222222222,storageclass=ebs-1-retain phase_type=3i 1547597616000000000 kubernetes_persistentvolumeclaim,namespace=default,phase=Bound,pvc_name=data-etcd-0,storageclass=ebs-1-retain phase_type=0i 1547597615000000000 kubernetes_pod,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1 last_transition_time=1547578322000000000i,ready="false" 1547597616000000000 -kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running,ready=ready resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,state_reason="",terminated_reason="",resource_requests_memory_bytes=524288000 1547597616000000000 +kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running,readiness=ready resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,state_reason="",resource_requests_memory_bytes=524288000 1547597616000000000 kubernetes_statefulset,namespace=default,statefulset_name=etcd replicas_updated=3i,spec_replicas=3i,observed_generation=1i,created=1544101669000000000i,generation=1i,replicas=3i,replicas_current=3i,replicas_ready=3i 1547597616000000000 ``` From e36048e6ae436469a6b21a9ca9d7a05796431cf7 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 16 Jun 2020 18:54:41 +0100 Subject: [PATCH 4/4] chore: fix up tests to reflect changes to plugin --- plugins/inputs/kube_inventory/pod_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/kube_inventory/pod_test.go b/plugins/inputs/kube_inventory/pod_test.go index 877cd5a045b76..aac29b9f134d7 100644 --- a/plugins/inputs/kube_inventory/pod_test.go +++ b/plugins/inputs/kube_inventory/pod_test.go @@ -209,7 +209,6 @@ func TestPod(t *testing.T) { Fields: map[string]interface{}{ "restarts_total": int32(3), "state_code": 0, - "state_reason": "", "resource_requests_millicpu_units": int64(100), "resource_limits_millicpu_units": int64(100), }, @@ -219,7 +218,7 @@ func TestPod(t *testing.T) { "node_name": "node1", "pod_name": "pod1", "state": "running", - "ready": "ready", + "readiness": "ready", }, }, { @@ -237,7 +236,7 @@ func TestPod(t *testing.T) { "node_name": "node1", "pod_name": "pod1", "state": "terminated", - "ready": "unready", + "readiness": "unready", }, }, { @@ -255,7 +254,7 @@ func TestPod(t *testing.T) { "node_name": "node1", "pod_name": "pod1", "state": "waiting", - "ready": "unready", + "readiness": "unready", }, }, },