Skip to content

Commit

Permalink
Add state and readiness to kube_inventory pod metrics (influxdata#7691)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rawkode
rawkode authored and idohalevi committed Sep 23, 2020
1 parent c300527 commit 3d68238
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 9 deletions.
13 changes: 9 additions & 4 deletions plugins/inputs/kube_inventory/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ avoid cardinality issues:
# tls_key = "/path/to/keyfile"
## Use TLS but skip chain & host verification
# insecure_skip_verify = false

## Uncomment to remove deprecated metrics.
# fielddrop = ["terminated_reason"]
```

#### Kubernetes Permissions
Expand Down Expand Up @@ -206,10 +209,13 @@ subjects:
- namespace
- node_name
- pod_name
- state
- readiness
- fields:
- restarts_total
- state
- terminated_reason
- state_code
- state_reason
- terminated_reason (string, deprecated in 1.15: use `state_reason` instead)
- resource_requests_cpu_units
- resource_requests_memory_bytes
- resource_limits_cpu_units
Expand Down Expand Up @@ -267,7 +273,6 @@ The persistentvolumeclaim "phase" is saved in the `phase` tag with a correlated
| pending | 2 |
| unknown | 3 |


### Example Output:

```
Expand All @@ -278,7 +283,7 @@ kubernetes_node,node_name=ip-172-17-0-2.internal allocatable_pods=110i,capacity_
kubernetes_persistentvolume,phase=Released,pv_name=pvc-aaaaaaaa-bbbb-cccc-1111-222222222222,storageclass=ebs-1-retain phase_type=3i 1547597616000000000
kubernetes_persistentvolumeclaim,namespace=default,phase=Bound,pvc_name=data-etcd-0,storageclass=ebs-1-retain phase_type=0i 1547597615000000000
kubernetes_pod,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1 last_transition_time=1547578322000000000i,ready="false" 1547597616000000000
kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,terminated_reason="",resource_requests_memory_bytes=524288000 1547597616000000000
kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running,readiness=ready resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,state_reason="",resource_requests_memory_bytes=524288000 1547597616000000000
kubernetes_statefulset,namespace=default,statefulset_name=etcd replicas_updated=3i,spec_replicas=3i,observed_generation=1i,created=1544101669000000000i,generation=1i,replicas=3i,replicas_current=3i,replicas_ready=3i 1547597616000000000
```
Expand Down
17 changes: 16 additions & 1 deletion plugins/inputs/kube_inventory/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package kube_inventory
import (
"context"

"github.com/ericchiang/k8s/apis/core/v1"
v1 "github.com/ericchiang/k8s/apis/core/v1"

"github.com/influxdata/telegraf"
)
Expand Down Expand Up @@ -37,30 +37,45 @@ func (ki *KubernetesInventory) gatherPod(p v1.Pod, acc telegraf.Accumulator) err

func gatherPodContainer(nodeName string, p v1.Pod, cs v1.ContainerStatus, c v1.Container, acc telegraf.Accumulator) {
stateCode := 3
stateReason := ""
state := "unknown"

switch {
case cs.State.Running != nil:
stateCode = 0
state = "running"
case cs.State.Terminated != nil:
stateCode = 1
state = "terminated"
stateReason = cs.State.Terminated.GetReason()
case cs.State.Waiting != nil:
stateCode = 2
state = "waiting"
stateReason = cs.State.Waiting.GetReason()
}

readiness := "unready"
if cs.GetReady() {
readiness = "ready"
}

fields := map[string]interface{}{
"restarts_total": cs.GetRestartCount(),
"state_code": stateCode,
"terminated_reason": cs.State.Terminated.GetReason(),
}

if stateReason != "" {
fields["state_reason"] = stateReason
}

tags := map[string]string{
"container_name": *c.Name,
"namespace": *p.Metadata.Namespace,
"node_name": *p.Spec.NodeName,
"pod_name": *p.Metadata.Name,
"state": state,
"readiness": readiness,
}

req := c.Resources.Requests
Expand Down
109 changes: 105 additions & 4 deletions plugins/inputs/kube_inventory/pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"testing"
"time"

"github.com/ericchiang/k8s/apis/core/v1"
v1 "github.com/ericchiang/k8s/apis/core/v1"
metav1 "github.com/ericchiang/k8s/apis/meta/v1"
"github.com/ericchiang/k8s/apis/resource"
"github.com/influxdata/telegraf/testutil"
Expand Down Expand Up @@ -44,7 +44,43 @@ func TestPod(t *testing.T) {
NodeName: toStrPtr("node1"),
Containers: []*v1.Container{
{
Name: toStrPtr("forwarder"),
Name: toStrPtr("running"),
Image: toStrPtr("image1"),
Ports: []*v1.ContainerPort{
{
ContainerPort: toInt32Ptr(8080),
Protocol: toStrPtr("TCP"),
},
},
Resources: &v1.ResourceRequirements{
Limits: map[string]*resource.Quantity{
"cpu": {String_: toStrPtr("100m")},
},
Requests: map[string]*resource.Quantity{
"cpu": {String_: toStrPtr("100m")},
},
},
},
{
Name: toStrPtr("completed"),
Image: toStrPtr("image1"),
Ports: []*v1.ContainerPort{
{
ContainerPort: toInt32Ptr(8080),
Protocol: toStrPtr("TCP"),
},
},
Resources: &v1.ResourceRequirements{
Limits: map[string]*resource.Quantity{
"cpu": {String_: toStrPtr("100m")},
},
Requests: map[string]*resource.Quantity{
"cpu": {String_: toStrPtr("100m")},
},
},
},
{
Name: toStrPtr("waiting"),
Image: toStrPtr("image1"),
Ports: []*v1.ContainerPort{
{
Expand Down Expand Up @@ -101,7 +137,7 @@ func TestPod(t *testing.T) {
},
ContainerStatuses: []*v1.ContainerStatus{
{
Name: toStrPtr("forwarder"),
Name: toStrPtr("running"),
State: &v1.ContainerState{
Running: &v1.ContainerStateRunning{
StartedAt: &metav1.Time{Seconds: toInt64Ptr(cond2.Unix())},
Expand All @@ -113,6 +149,34 @@ func TestPod(t *testing.T) {
ImageID: toStrPtr("image_id1"),
ContainerID: toStrPtr("docker://54abe32d0094479d3d"),
},
{
Name: toStrPtr("completed"),
State: &v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
StartedAt: &metav1.Time{Seconds: toInt64Ptr(cond2.Unix())},
ExitCode: toInt32Ptr(0),
Reason: toStrPtr("Completed"),
},
},
Ready: toBoolPtr(false),
RestartCount: toInt32Ptr(3),
Image: toStrPtr("image1"),
ImageID: toStrPtr("image_id1"),
ContainerID: toStrPtr("docker://54abe32d0094479d3d"),
},
{
Name: toStrPtr("waiting"),
State: &v1.ContainerState{
Waiting: &v1.ContainerStateWaiting{
Reason: toStrPtr("PodUninitialized"),
},
},
Ready: toBoolPtr(false),
RestartCount: toInt32Ptr(3),
Image: toStrPtr("image1"),
ImageID: toStrPtr("image_id1"),
ContainerID: toStrPtr("docker://54abe32d0094479d3d"),
},
},
},
Metadata: &metav1.ObjectMeta{
Expand Down Expand Up @@ -150,10 +214,47 @@ func TestPod(t *testing.T) {
},
Tags: map[string]string{
"namespace": "ns1",
"container_name": "forwarder",
"container_name": "running",
"node_name": "node1",
"pod_name": "pod1",
"state": "running",
"readiness": "ready",
},
},
{
Measurement: podContainerMeasurement,
Fields: map[string]interface{}{
"restarts_total": int32(3),
"state_code": 1,
"state_reason": "Completed",
"resource_requests_millicpu_units": int64(100),
"resource_limits_millicpu_units": int64(100),
},
Tags: map[string]string{
"namespace": "ns1",
"container_name": "completed",
"node_name": "node1",
"pod_name": "pod1",
"state": "terminated",
"readiness": "unready",
},
},
{
Measurement: podContainerMeasurement,
Fields: map[string]interface{}{
"restarts_total": int32(3),
"state_code": 2,
"state_reason": "PodUninitialized",
"resource_requests_millicpu_units": int64(100),
"resource_limits_millicpu_units": int64(100),
},
Tags: map[string]string{
"namespace": "ns1",
"container_name": "waiting",
"node_name": "node1",
"pod_name": "pod1",
"state": "waiting",
"readiness": "unready",
},
},
},
Expand Down

0 comments on commit 3d68238

Please sign in to comment.