Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[O11y][Kubernetes] Rally benchmark kubernetes.state_container #9106

Merged
merged 8 commits into from
Feb 15, 2024
Merged

[O11y][Kubernetes] Rally benchmark kubernetes.state_container #9106

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
07c98bb
rally benchmark kubernetes.state_container
ali786XI Feb 9, 2024
a9ad8d0
add container name
ali786XI Feb 9, 2024
988e6c4
address review comments
ali786XI Feb 12, 2024
8c02a6f
add last_terminated_reason
ali786XI Feb 12, 2024
27d02ea
rename yml file
ali786XI Feb 12, 2024
4da0f68
rename yml files for aws package
ali786XI Feb 12, 2024
c821c27
rename yml files for mysql package
ali786XI Feb 12, 2024
74a8e64
remove host.ip and host.mac
ali786XI Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/statecontainer-benchmark.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
description: Benchmark 20000 kubernetes.state_container events ingested
data_stream:
name: state_container
corpora:
generator:
total_events: 20000
template:
type: gotext
path: ./statecontainer-benchmark/template.ndjson
config:
path: ./statecontainer-benchmark/config.yml
fields:
path: ./statecontainer-benchmark/fields.yml
45 changes: 45 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/statecontainer-benchmark/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
fields:
- name: agent_ephemeral_id
ali786XI marked this conversation as resolved.
Show resolved Hide resolved
value: 22ed892c-43bd-408a-9121-65e2f5b6a56e
- name: agent_id
value: de42127b-4db8-4471-824e-a7b14f478663
- name: agent_name
value: "kind-control-plane"
- name: timestamp
period: 60m
- name: event_duration
range:
min: 1
max: 1000000
- name: status_phase
enum: ["running", "waiting", "terminated"]
- name: status_ready
enum: ["true", "false"]
- name: restarts
range:
min: 1
max: 100
- name: reason
enum: ["ContainerCreating", "CrashLoopBackoff", "ErrImagePull", "ImagePullBackoff", "Completed", "ContainerCannotRun", "Error", "OOMKilled"]
- name: cpu_limit_cores
range:
min: 4
max: 6
- name: cpu_request_cores
range:
min: 0
max: 3
- name: memory_limit_bytes
range:
min: 1000000000
max: 16000000000
- name: memory_request_bytes
range:
min: 500000000
max: 999000000
- name: rangeofid
range:
min: 0
max: 10000
- name: container_name
enum: ["web", "default-http-backend", "dnsmasq", "csi-driver", "web", "web", "web", "prometheus", "konnectivity-agent", "sidecar", "kubedns", "metrics-server-nanny", "web", "web", "fluentbit", "autoscaler", "gke-metrics-agent", "elastic-agent", "web", "kube-state-metrics", "metrics-server", "fluentbit", "elastic-agent", "web", "prometheus-to-sd-exporter"]
32 changes: 32 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/statecontainer-benchmark/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
- name: Ip
type: ip
- name: timestamp
type: date
- name: agent_id
type: keyword
- name: agent_ephemeral_id
type: keyword
- name: agent_name
type: keyword
- name: rangeofid
type: integer
- name: event_duration
type: long
- name: container_name
type: keyword
- name: status_phase
type: keyword
- name: status_ready
type: boolean
- name: restarts
type: integer
- name: reason
type: keyword
- name: cpu_limit_cores
tommyers-elastic marked this conversation as resolved.
Show resolved Hide resolved
type: float
- name: cpu_request_cores
type: float
- name: memory_limit_bytes
type: long
- name: memory_request_bytes
type: long
171 changes: 171 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/statecontainer-benchmark/template.ndjson
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
{{- $timestamp := generate "timestamp" }}
{{- $agent_id := generate "agent_id" }}
{{- $agent_name := generate "agent_name" }}
{{- $agent_ephemeral_id := generate "agent_ephemeral_id" }}
{{- $event_duration := generate "event_duration" }}
{{- $rangeofid := generate "rangeofid" -}}
{{- $nodeid := div $rangeofid 110 -}}
{{- $container_name := generate "container_name" }}
{{- $status_phase := generate "status_phase" }}
{{- $status_ready := generate "status_ready" }}
{{- $restarts := generate "restarts" }}
{{- $reason := generate "reason" }}
{{- $cpu_limit_cores := generate "cpu_limit_cores" }}
{{- $cpu_request_cores := generate "cpu_request_cores" }}
{{- $memory_limit_bytes := generate "memory_limit_bytes" }}
{{- $memory_request_bytes := generate "memory_request_bytes" }}
{
"@timestamp": "{{$timestamp.Format "2006-01-02T15:04:05.999999Z07:00"}}",
"container": {
ali786XI marked this conversation as resolved.
Show resolved Hide resolved
"image": {
"name": "k8s.gcr.io/coredns/coredns:v1.8.0"
},
"name": "{{ $container_name }}",
"runtime": "containerd",
"id": "container-{{ $rangeofid }}"
},
"kubernetes": {
"container": {
"memory": {
"request": {
"bytes": {{ $memory_request_bytes }}
},
"limit": {
"bytes": {{ $memory_limit_bytes }}
}
},
"name": "{{ $container_name }}",
"cpu": {
"request": {
"cores": {{ $cpu_request_cores }}
},
"limit": {
"cores": {{ $cpu_limit_cores }}
}
},
"id": "container-{{ $rangeofid }}",
"status": {
"phase": "{{ $status_phase }}",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only values that can probably cause us problems in the future are these one for the status. Because the way it is implemented now a 'running' phase can have ready: false , because those values are randomly assigned.

Can you consider an if case like https://github.com/elastic/elastic-integration-corpus-generator-tool/blob/main/assets/templates/aws.billing/schema-b/gotext.tpl#L64 ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure. Updated the same. Please have a look.

"ready": {{ $status_ready }},
"restarts": {{ $restarts }},
"reason": "{{ $reason }}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"reason": "{{ $reason }}"
"last_terminated_reason": "{{ $reason }}"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Took reference from here. Found this field you are mentioning as kubernetes.container.status.last_terminated_reason

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am testing with a real cluster and just spotted that as well.
Screenshot 2024-02-12 at 3 11 00 PM

In beats: https://github.com/elastic/beats/blob/main/metricbeat/module/kubernetes/state_container/state_container.go#L69

Will have a second look in the background to explain the diff

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When container is in terminated or waiting state ,the status.reason is populated
When container is in running state, then the status kubernetes.container.status.last_terminated_reason is populated

See info here and here

So it is probably another if case for you, that when phase is running then populate this kubernetes.container.status.last_terminated_reason

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gizas Thanks for this. Added the same.

}
},
"node": {
"uid": "host-{{ $nodeid }}",
"hostname": "host-{{ $nodeid }}",
"name": "host-{{ $nodeid }}",
"labels": {
"kubernetes_io/hostname": "kind-control-plane",
"beta_kubernetes_io/os": "linux",
"kubernetes_io/arch": "amd64",
"kubernetes_io/os": "linux",
"beta_kubernetes_io/arch": "amd64"
}
},
"pod": {
"uid": "demo-pod-{{ $rangeofid }}",
"ip": "{{generate `Ip`}}",
"name": "demo-pod-{{ $rangeofid }}"
},
"namespace": "demo-{{ $rangeofid }}",
"namespace_uid": "demo-{{ $rangeofid }}",
"replicaset": {
"name": "demo-deployment-{{ $rangeofid }}"
},
"namespace_labels": {
"kubernetes_io/metadata_name": "demo-{{ $rangeofid }}"
},
"labels": {
"app":"demo",
"pod-template-hash":"{{ $rangeofid }}",
"app-2":"demo-2",
"app-1":"demo-1"
},
"deployment": {
"name": "demo-deployment-{{ $rangeofid }}"
}
},
"agent": {
"name": "{{ $agent_name }}",
"id": "{{ $agent_id }}",
"ephemeral_id": "{{ $agent_ephemeral_id }}",
"type": "metricbeat",
"version": "8.8.0"
},
"elastic_agent": {
"id": "{{ $agent_id }}",
"version": "8.8.0",
"snapshot": true
},
"cloud": {
"provider": "gcp",
"availability_zone": "europe-west1-d",
"instance":{
"name": "{{ $agent_name }}" ,
"id": "{{ $agent_id }}"
},
"machine":{
"type":"e2-standard-4"
},
"service":{
"name":"GCE"
},
"project":{
"id":"elastic-obs-integrations-dev"
},
"account":{
"id":"elastic-obs-integrations-dev"
}
},
"orchestrator":{
"cluster":{
"name":"kubernetes-scale",
"url":"https://{{ generate `Ip` }}"
}
},
"ecs": {
"version": "8.0.0"
},
"data_stream": {
"namespace": "ep",
"type": "metrics",
"dataset": "kubernetes.state_container"
},
"service": {
"address": "http://{{ $agent_name }}:8080/metrics",
"type": "kubernetes"
},
"host": {
"hostname": "kind-control-plane",
"os": {
"kernel": "5.10.47-linuxkit",
"codename": "Core",
"name": "CentOS Linux",
"type": "linux",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos"
},
"containerized": true,
"ip": [
ali786XI marked this conversation as resolved.
Show resolved Hide resolved
"192.168.244.7"
],
"name": "kind-control-plane",
"id": "85e35c2b5e1b39ba72393a6baf6ee7cd",
"mac": [
"fe:ec:82:9f:29:19"
],
"architecture": "x86_64"
},
"metricset": {
"period": 10000,
"name": "state_container"
},
"event": {
"duration": {{ $event_duration }},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ingested fields is missing
eg. event.ingested: "2024-02-08T12:18:41Z"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please refer here for the same doubt I had. It's taking the time actually when the event is ingested.

"agent_id_status": "verified",
"module": "kubernetes",
"dataset": "kubernetes.state_container"
}
}