Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[O11y][Kubernetes] Rally benchmark kubernetes.node #9100

Merged
merged 4 commits into from
Feb 26, 2024
Merged

[O11y][Kubernetes] Rally benchmark kubernetes.node #9100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
61d80c4
rally benchmark kubernetes.node
ali786XI Feb 8, 2024
0bcd90e
revert changes
ali786XI Feb 8, 2024
5e3993b
add some fields
ali786XI Feb 14, 2024
ad2d1bb
remove unnecessary fields and add duration for start time
ali786XI Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/node-benchmark.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
description: Benchmark 20000 kubernetes.node events ingested
data_stream:
name: node
corpora:
generator:
total_events: 20000
template:
type: gotext
path: ./node-benchmark/template.ndjson
config:
path: ./node-benchmark/config.yml
fields:
path: ./node-benchmark/fields.yml
97 changes: 97 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/node-benchmark/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
fields:
- name: timestamp
period: 60m
- name: rangeofid
range:
min: 0
max: 10000
- name: duration_start
enum: ["-1h", "-2h", "-3h", "-4h"]
- name: event_duration
range:
min: 1
max: 1000000
- name: cpu_usage_core_ns
range:
min: 1
max: 1000000000000
- name: nanocores
range:
min: 1
max: 1000000
- name: memory_available_bytes
range:
min: 100000000001
max: 200000000000
- name: memory_usage_bytes
range:
min: 10000000001
max: 90000000000
- name: memory_rss_bytes
range:
min: 1
max: 1000000000
- name: memory_workingset_bytes
range:
min: 1000000000
max: 9000000000
- name: pagefaults
range:
min: 1
max: 100000
- name: majorpagefaults
range:
min: 0
max: 100
- name: network_rx_bytes
range:
min: 100000000
max: 900000000
- name: network_rx_errors
range:
min: 0
max: 100
- name: network_tx_bytes
range:
min: 10000000
max: 90000000
- name: network_tx_errors
range:
min: 0
max: 100
- name: fs_capacity_bytes
range:
min: 10000000000
max: 20000000000
- name: fs_available_bytes
range:
min: 1000000000
max: 9000000000
- name: fs_used_bytes
range:
min: 1000000000
max: 9000000000
- name: fs_inodes_used
range:
min: 100000
max: 900000
- name: fs_inodes_count
range:
min: 1000000
max: 9000000
- name: fs_inodes_free
range:
min: 1000000
max: 9000000
- name: runtime_imagefs_capacity_bytes
range:
min: 10000000000
max: 20000000000
- name: runtime_imagefs_available_bytes
range:
min: 1000000000
max: 9000000000
- name: runtime_imagefs_used_bytes
range:
min: 1000000000
max: 9000000000
56 changes: 56 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/node-benchmark/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
- name: Ip
type: ip
- name: timestamp
type: date
- name: rangeofid
type: integer
- name: event_duration
type: long
- name: duration_start
type: keyword
- name: cpu_usage_core_ns
type: long
- name: nanocores
type: long
- name: cpu_usage_core_ns
type: long
- name: nanocores
type: long
- name: memory_available_bytes
type: long
- name: memory_usage_bytes
type: long
- name: memory_rss_bytes
type: long
- name: memory_workingset_bytes
type: long
- name: pagefaults
type: long
- name: majorpagefaults
type: long
- name: network_rx_bytes
type: long
- name: network_rx_errors
type: long
- name: network_tx_bytes
type: long
- name: network_tx_errors
type: long
- name: fs_capacity_bytes
type: long
- name: fs_available_bytes
type: long
- name: fs_used_bytes
type: long
- name: fs_inodes_used
type: long
- name: fs_inodes_count
type: long
- name: fs_inodes_free
type: long
- name: runtime_imagefs_capacity_bytes
type: long
- name: runtime_imagefs_available_bytes
type: long
- name: runtime_imagefs_used_bytes
type: long
160 changes: 160 additions & 0 deletions packages/kubernetes/_dev/benchmark/rally/node-benchmark/template.ndjson
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
{{- $timestamp := generate "timestamp" }}
{{- $event_duration := generate "event_duration" }}
{{- $rangeofid := generate "rangeofid" -}}
{{- $nodeid := div $rangeofid 110 -}}
{{- $duration_start := generate "duration_start" }}
{{- $startTime := generate "timestamp" | date_modify (print $duration_start) }}
{{- $memory_available_bytes := generate "memory_available_bytes" }}
{{- $memory_usage_bytes := generate "memory_usage_bytes" }}
{{- $memory_rss_bytes := generate "memory_rss_bytes" }}
{{- $memory_workingset_bytes := generate "memory_workingset_bytes" }}
{{- $pagefaults := generate "pagefaults" }}
{{- $majorpagefaults := generate "majorpagefaults" }}
{{- $network_rx_bytes := generate "network_rx_bytes" }}
{{- $network_rx_errors := generate "network_rx_errors" }}
{{- $network_tx_bytes := generate "network_tx_bytes" }}
{{- $network_tx_errors := generate "network_tx_errors" }}
{{- $fs_capacity_bytes := generate "fs_capacity_bytes" }}
{{- $fs_available_bytes := generate "fs_available_bytes" }}
{{- $fs_used_bytes := generate "fs_used_bytes" }}
{{- $fs_inodes_used := generate "fs_inodes_used" }}
{{- $fs_inodes_count := generate "fs_inodes_count" }}
{{- $fs_inodes_free := generate "fs_inodes_free" }}
{{- $runtime_imagefs_capacity_bytes := generate "runtime_imagefs_capacity_bytes" }}
{{- $runtime_imagefs_available_bytes := generate "runtime_imagefs_available_bytes" }}
{{- $runtime_imagefs_used_bytes := generate "runtime_imagefs_used_bytes" }}
{{- $cpu_usage_core_ns := generate "cpu_usage_core_ns" }}
{{- $nanocores := generate "nanocores" }}
{
"@timestamp": "{{$timestamp.Format "2006-01-02T15:04:05.999999Z07:00"}}",
"event": {
"dataset": "kubernetes.node",
"module": "kubernetes",
"agent_id_status": "verified",
"duration": {{ $event_duration }}
},
"data_stream": {
"dataset": "kubernetes.node",
"namespace": "ep",
"type": "metrics"
},
"service": {
"type": "kubernetes",
"address": "kubernetes-scale-123456:10250"
},
"host": {
"hostname": "kubernetes-scale-123456",
"os": {
"kernel": "5.10.47-linuxkit",
"codename": "Core",
"name": "CentOS Linux",
"type": "linux",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos"
},
"containerized": true,
"name": "kubernetes-scale-123456",
"id": "85e35c2b5e1b39ba72393a6baf6ee7cd",
"architecture": "x86_64"
},
"metricset": {
"name": "node",
"period": 10000
},
"kubernetes": {
"labels": {
"app":"demo",
"pod-template-hash":"{{ $rangeofid }}",
"app-2":"demo-2",
"app-1":"demo-1"
},
"node": {
"memory": {
"available": {
"bytes": {{ $memory_available_bytes }}
},
"usage": {
"bytes": {{ $memory_usage_bytes }}
},
"workingset": {
"bytes": {{ $memory_workingset_bytes }}
},
"rss": {
"bytes": {{ $memory_rss_bytes }}
},
"pagefaults": {{ $pagefaults }},
"majorpagefaults": {{ $majorpagefaults }}
},
"network": {
"rx": {
"bytes": {{ $network_rx_bytes }},
"errors": {{ $network_rx_errors }}
},
"tx": {
"bytes": {{ $network_tx_bytes }},
"errors": {{ $network_tx_errors }}
}
},
"fs": {
"available": {
"bytes": {{ $fs_available_bytes }}
},
"capacity": {
"bytes": {{ $fs_capacity_bytes }}
},
"used": {
"bytes": {{ $fs_used_bytes }}
},
"inodes": {
"count": {{ $fs_inodes_count }},
"used": {{ $fs_inodes_used }},
"free": {{ $fs_inodes_free }}
}
},
"runtime": {
"imagefs": {
"capacity": {
"bytes": {{ $runtime_imagefs_capacity_bytes }}
},
"used": {
"bytes": {{ $runtime_imagefs_used_bytes }}
},
"available": {
"bytes": {{ $runtime_imagefs_available_bytes }}
}
}
},
"start_time": "{{$startTime.Format "2006-01-02T15:04:05.000Z"}}",
"cpu": {
"usage": {
"core": {
"ns": {{ $cpu_usage_core_ns }}
},
"nanocores": {{ $nanocores }}
}
}
}
},
"agent": {
"name": "kubernetes-scale-123456",
"type": "metricbeat",
"version": "8.8.0",
"ephemeral_id": "22ed892c-43bd-408a-9121-65e2f5b6a56e",
"id": "de42127b-4db8-4471-824e-a7b14f478663"
},
"elastic_agent": {
"id": "de42127b-4db8-4471-824e-a7b14f478663",
"version": "8.8.0",
"snapshot": true
},
"orchestrator":{
"cluster":{
"name":"kubernetes-scale",
"url":"https://{{ generate `Ip` }}"
}
},
"ecs": {
"version": "8.5.0"
}
}