Skip to content

Commit

Permalink
[Obs][kbn-data-forge] Adding example config (elastic#176727)
Browse files Browse the repository at this point in the history
## Summary

This PR copies the example configurations from the High Cardinality
Indexer project into `kbn-data-forge`

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
2 people authored and fkanout committed Mar 4, 2024
1 parent 1c9a375 commit bc3fa49
Show file tree
Hide file tree
Showing 32 changed files with 1,133 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
---
elasticsearch:
installKibanaUser: false

kibana:
installAssets: false

indexing:
eventsPerCycle: 5000
dataset: "fake_logs"

schedule:
# Start with normal logs
- template: "good"
start: "now-45m"
end: "now+1m"
randomness: 0.1
eventsPerCycle: 5000
# Sudden change into new number of logs
- template: "good"
start: "now+1m"
end: "now+10m"
randomness: 0.1
eventsPerCycle: 1000


Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
---
elasticsearch:
installKibanaUser: false

kibana:
installAssets: false

indexing:
eventsPerCycle: 1000
dataset: "fake_logs"

schedule:
# Start with normal logs
- template: "good"
start: "now-10m"
end: "now+1m"
randomness: 0.1
eventsPerCycle: 1000
# Progresively change into a new number of logs
- template: "good"
start: "now+1m"
end: "now+5m"
randomness: 0.1
eventsPerCycle:
start: 1000
end: 5000
method: "linear"
# Stay at the new number of logs
- template: "good"
start: "now+5m"
end: "now+10m"
randomness: 0.1
eventsPerCycle: 5000


Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
---
# Define the connection to Elasticsearch
elasticsearch:
installKibanaUser: false

# Define the connection to Kibana
kibana:
host: "http://localhost:5601"
username: "elastic"
password: "changeme"
installAssets: false

# Define the indexing options
indexing:
dataset: "fake_logs"
eventsPerCycle: 2000
interval: 6000

# Define the schedule
schedule:
- template: "bad"
start: "now-5m"
end: "now+1m"
eventsPerCycle:
start: 5000
end: 3000
method: "sine"
- template: "good"
start: "now+1m"
end: "now+2m"
eventsPerCycle: 3000
- template: "bad"
start: "now+2m"
end: "now+10m"
eventsPerCycle:
start: 5000
end: 3000
method: "sine"
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
---
# Define the connection to Elasticsearch
elasticsearch:
installKibanaUser: false

# Define the connection to Kibana
kibana:
installAssets: false

# Define the indexing options
indexing:
dataset: "fake_logs"
eventsPerCycle: 5000

# Define the schedule
schedule:
# Normal logs
- template: "good"
eventsPerCycle: 5000
start: "now-5m"
end: "now+5m"
randomness: 0.1
# Spike in logs
- template: "bad"
eventsPerCycle: 10000
start: "now+1m"
end: "now+2m"
randomness: 0.1
# Drop in logs, stop normal logs and send few logs
- template: "bad"
eventsPerCycle: 500
start: "now+5m"
end: "now+7m"
randomness: 0.1
# Return to normal logs
- template: "good"
eventsPerCycle: 5000
start: "now+7m"
end: "now+17m"
randomness: 0.1
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
---
elasticsearch:
installKibanaUser: false

kibana:
installAssets: true

indexing:
eventsPerCycle: 1000
dataset: "fake_logs"

schedule:
- template: "good"
start: "now-2h"
end: "now-1h"
- template: "bad"
start: "now-1h"
end: "now-15m"
eventsPerCycle: 5000
- template: "good"
start: "now-15m"
end: false

31 changes: 31 additions & 0 deletions x-pack/packages/kbn-data-forge/example_config/fake_logs_sine.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
---
# Define the connection to Elasticsearch
elasticsearch:
installKibanaUser: false

# Define the connection to Kibana
kibana:
host: "http://localhost:5601"
username: "elastic"
password: "changeme"
installAssets: false

# Define the indexing options
indexing:
dataset: "fake_logs"
eventsPerCycle: 2000
interval: 6000
scenario: "sine_logs"

# Define the schedule
schedule:
- template: "bad"
start: "now-5m"
end: "now+10m"
eventsPerCycle:
start: 2000
end: 3000
method: "sine"
options:
period: 60
randomness: 0.1
39 changes: 39 additions & 0 deletions x-pack/packages/kbn-data-forge/example_config/fake_stack.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
---
elasticsearch:
installKibanaUser: false

kibana:
installAssets: true

indexing:
dataset: "fake_stack"
eventsPerCycle: 200
reduceWeekendTrafficBy: 0.5

schedule:
# Start with good events
- template: "good"
start: "now-14d"
end: "now-6d-5h-3m"
eventsPerCycle: 200
randomness: 0.2
- template: "connectionTimeout"
start: "now-6d-5h-3m"
end: "now-6d-4h-44m"
eventsPerCycle: 200
randomness: 0.2
- template: "good"
start: "now-6d-4h-44m"
end: "now-1d"
eventsPerCycle: 200
randomness: 0.2
- template: "bad"
start: "now-1d"
end: "now-1d+45m"
eventsPerCycle: 200
randomness: 0.2
- template: "good"
start: "now-1d+45m"
end: false
eventsPerCycle: 200
randomness: 0.2
41 changes: 41 additions & 0 deletions x-pack/packages/kbn-data-forge/example_config/full_example.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
---
# Define the connection to Elasticsearch
elasticsearch:
host: "http://localhost:9200"
username: "elastic"
password: "changeme"
installKibanaUser: false

# Define the connection to Kibana
kibana:
host: "http://localhost:5601"
username: "elastic"
password: "changeme"
installAssets: true

# Define the indexing options
indexing:
dataset: "fake_stack"
interval: 6000
eventsPerCycle: 1
payloadSize: 10000
concurrency: 5

# Define the schedule
schedule:
- template: "good"
start: "now-1h"
end: "now-15m"
- template: "bad"
start: "now-15m"
end: "now"
# This schedule will run indefinitely
- template: "good"
start: "now"
end: false
# This will add a 2 minute delay to the indexing every 5 mintes.
# Once the 2 minutes is up, the queue will back fill the events it collected
# during the delay. This only makes sense if `end` is `false`
delayInMinutes: 2
delayEveryMinutes: 5

20 changes: 20 additions & 0 deletions x-pack/packages/kbn-data-forge/example_config/future_example.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
---
elasticsearch:
installKibanaUser: false

kibana:
installAssets: true

indexing:
dataset: "fake_stack"

schedule:
- template: "good"
start: "now"
end: "now+1h"
- template: "bad"
start: "now+1h"
end: "now+90m"
- template: "good"
start: "now+90m"
end: "now+2h"
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
---
elasticsearch:
installKibanaUser: false

kibana:
installAssets: true

indexing:
dataset: "fake_stack"

schedule:
- template: "good"
start: "now-2h"
end: "now-1h"
- template: "bad"
start: "now-1h"
end: "now-15m"
- template: "good"
start: "now-15m"
end: false
39 changes: 39 additions & 0 deletions x-pack/packages/kbn-data-forge/example_config/log_drop.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
---
# Define the connection to Elasticsearch
elasticsearch:
installKibanaUser: false

# Define the connection to Kibana
kibana:
host: "http://localhost:5601"
username: "elastic"
password: "changeme"
installAssets: false

indexing:
eventsPerCycle: 100
dataset: "fake_logs"
scenario: "log_drop"

schedule:
# Start with good events at 100 rate
- template: "good"
start: "now-30m"
end: "now-5m"
eventsPerCycle: 50
randomness: 0.1
# Create a ramp-up of bad events
- template: "good"
start: "now-5m"
end: "now+5m"
eventsPerCycle:
start: 50
end: 100
method: "linear"
randomness: 0.1
# Drop to very little good events
- template: "good"
start: "now+5m"
end: false
eventsPerCycle: 10
randomness: 0.1
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
# Define the connection to Elasticsearch
elasticsearch:
host: "http://localhost:9200"
username: "elastic"
password: "changeme"
installKibanaUser: false

# Define the connection to Kibana
kibana:
host: "http://localhost:5601"
username: "elastic"
password: "changeme"
installAssets: false

# Define the indexing options
indexing:
dataset: "fake_logs"
eventsPerCycle: 2000
interval: 6000

# Define the schedule
schedule:
# This step send 2000 logs every 30 seconds
- template: "good"
eventsPerCycle: 2000
start: "now-5m"
end: "now+5m"

Loading

0 comments on commit bc3fa49

Please sign in to comment.