From e44676e31c8fec8b5d26a44dfe646ef5dcd9c063 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 13 Feb 2024 08:30:00 -0700 Subject: [PATCH] [Obs][kbn-data-forge] Adding example config (#176727) ## Summary This PR copies the example configurations from the High Cardinality Indexer project into `kbn-data-forge` Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../change_point_detection.yaml | 26 ++++++++ .../anomalies_by_type/concept_drift.yaml | 35 ++++++++++ .../anomalies_by_type/contextual_anomaly.yaml | 38 +++++++++++ .../anomalies_by_type/point_anomaly.yaml | 40 +++++++++++ .../changing_log_volume_example.yaml | 23 +++++++ .../example_config/fake_logs_sine.yaml | 31 +++++++++ .../example_config/fake_stack.yaml | 39 +++++++++++ .../example_config/full_example.yaml | 41 ++++++++++++ .../example_config/future_example.yaml | 20 ++++++ .../example_config/good_to_bad_to_good.yaml | 20 ++++++ .../example_config/log_drop.yml | 39 +++++++++++ .../log_spike_scenarios/scenario0_logs.yaml | 29 ++++++++ .../scenario1_spike_logs.yaml | 35 ++++++++++ .../scenario2_spike_logs_host.yaml | 35 ++++++++++ .../scenario3_spike_errors.yaml | 33 ++++++++++ .../scenario4_spike_errors_with_recovery.yaml | 37 +++++++++++ .../scenario5_spike_logs_linear.yaml | 40 +++++++++++ .../example_config/metric_example.yaml | 31 +++++++++ .../example_config/ramp_up_then_down.yaml | 59 +++++++++++++++++ ...io0_paralell_metrics_drop_step_change.yaml | 26 ++++++++ ...paralell_metrics_increase_step_change.yaml | 35 ++++++++++ .../scenario2_divergent_metrics.yaml | 43 ++++++++++++ .../scenario3_chained_metrics_change.yaml | 59 +++++++++++++++++ .../custom_threshold_log_count.yaml | 27 ++++++++ .../custom_threshold_log_count_groupby.yaml | 27 ++++++++ .../custom_threshold_log_count_nodata.yaml | 23 +++++++ .../custom_threshold_metric_avg.yaml | 44 +++++++++++++ .../custom_threshold_metric_avg_groupby.yaml | 45 +++++++++++++ .../custom_threshold_metric_avg_nodata.yaml | 20 ++++++ .../rule_tests/slo_burn_rate.yaml | 22 +++++++ .../example_config/transition_example.yaml | 45 +++++++++++++ .../transitioning_templates_example.yaml | 66 +++++++++++++++++++ 32 files changed, 1133 insertions(+) create mode 100644 x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/change_point_detection.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/concept_drift.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/contextual_anomaly.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/point_anomaly.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/changing_log_volume_example.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/fake_logs_sine.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/fake_stack.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/full_example.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/future_example.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/good_to_bad_to_good.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/log_drop.yml create mode 100644 x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario0_logs.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario1_spike_logs.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario2_spike_logs_host.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario3_spike_errors.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario4_spike_errors_with_recovery.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario5_spike_logs_linear.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/metric_example.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/ramp_up_then_down.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario0_paralell_metrics_drop_step_change.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario1_paralell_metrics_increase_step_change.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario2_divergent_metrics.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario3_chained_metrics_change.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_groupby.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_nodata.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_groupby.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_nodata.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/rule_tests/slo_burn_rate.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/transition_example.yaml create mode 100644 x-pack/packages/kbn-data-forge/example_config/transitioning_templates_example.yaml diff --git a/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/change_point_detection.yaml b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/change_point_detection.yaml new file mode 100644 index 0000000000000..034bad86a0bec --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/change_point_detection.yaml @@ -0,0 +1,26 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + eventsPerCycle: 5000 + dataset: "fake_logs" + +schedule: + # Start with normal logs + - template: "good" + start: "now-45m" + end: "now+1m" + randomness: 0.1 + eventsPerCycle: 5000 + # Sudden change into new number of logs + - template: "good" + start: "now+1m" + end: "now+10m" + randomness: 0.1 + eventsPerCycle: 1000 + + diff --git a/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/concept_drift.yaml b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/concept_drift.yaml new file mode 100644 index 0000000000000..3f4184bad949d --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/concept_drift.yaml @@ -0,0 +1,35 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + eventsPerCycle: 1000 + dataset: "fake_logs" + +schedule: + # Start with normal logs + - template: "good" + start: "now-10m" + end: "now+1m" + randomness: 0.1 + eventsPerCycle: 1000 + # Progresively change into a new number of logs + - template: "good" + start: "now+1m" + end: "now+5m" + randomness: 0.1 + eventsPerCycle: + start: 1000 + end: 5000 + method: "linear" + # Stay at the new number of logs + - template: "good" + start: "now+5m" + end: "now+10m" + randomness: 0.1 + eventsPerCycle: 5000 + + diff --git a/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/contextual_anomaly.yaml b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/contextual_anomaly.yaml new file mode 100644 index 0000000000000..bee9bc319631e --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/contextual_anomaly.yaml @@ -0,0 +1,38 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_logs" + eventsPerCycle: 2000 + interval: 6000 + +# Define the schedule +schedule: + - template: "bad" + start: "now-5m" + end: "now+1m" + eventsPerCycle: + start: 5000 + end: 3000 + method: "sine" + - template: "good" + start: "now+1m" + end: "now+2m" + eventsPerCycle: 3000 + - template: "bad" + start: "now+2m" + end: "now+10m" + eventsPerCycle: + start: 5000 + end: 3000 + method: "sine" diff --git a/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/point_anomaly.yaml b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/point_anomaly.yaml new file mode 100644 index 0000000000000..b3f91a5f18e79 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/anomalies_by_type/point_anomaly.yaml @@ -0,0 +1,40 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + installKibanaUser: false + +# Define the connection to Kibana +kibana: + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_logs" + eventsPerCycle: 5000 + +# Define the schedule +schedule: + # Normal logs + - template: "good" + eventsPerCycle: 5000 + start: "now-5m" + end: "now+5m" + randomness: 0.1 + # Spike in logs + - template: "bad" + eventsPerCycle: 10000 + start: "now+1m" + end: "now+2m" + randomness: 0.1 + # Drop in logs, stop normal logs and send few logs + - template: "bad" + eventsPerCycle: 500 + start: "now+5m" + end: "now+7m" + randomness: 0.1 + # Return to normal logs + - template: "good" + eventsPerCycle: 5000 + start: "now+7m" + end: "now+17m" + randomness: 0.1 diff --git a/x-pack/packages/kbn-data-forge/example_config/changing_log_volume_example.yaml b/x-pack/packages/kbn-data-forge/example_config/changing_log_volume_example.yaml new file mode 100644 index 0000000000000..5e71a2f8a11d8 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/changing_log_volume_example.yaml @@ -0,0 +1,23 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + eventsPerCycle: 1000 + dataset: "fake_logs" + +schedule: + - template: "good" + start: "now-2h" + end: "now-1h" + - template: "bad" + start: "now-1h" + end: "now-15m" + eventsPerCycle: 5000 + - template: "good" + start: "now-15m" + end: false + diff --git a/x-pack/packages/kbn-data-forge/example_config/fake_logs_sine.yaml b/x-pack/packages/kbn-data-forge/example_config/fake_logs_sine.yaml new file mode 100644 index 0000000000000..49c4ed4f7f2ef --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/fake_logs_sine.yaml @@ -0,0 +1,31 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_logs" + eventsPerCycle: 2000 + interval: 6000 + scenario: "sine_logs" + +# Define the schedule +schedule: + - template: "bad" + start: "now-5m" + end: "now+10m" + eventsPerCycle: + start: 2000 + end: 3000 + method: "sine" + options: + period: 60 + randomness: 0.1 diff --git a/x-pack/packages/kbn-data-forge/example_config/fake_stack.yaml b/x-pack/packages/kbn-data-forge/example_config/fake_stack.yaml new file mode 100644 index 0000000000000..8397c37f039fc --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/fake_stack.yaml @@ -0,0 +1,39 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_stack" + eventsPerCycle: 200 + reduceWeekendTrafficBy: 0.5 + +schedule: + # Start with good events + - template: "good" + start: "now-14d" + end: "now-6d-5h-3m" + eventsPerCycle: 200 + randomness: 0.2 + - template: "connectionTimeout" + start: "now-6d-5h-3m" + end: "now-6d-4h-44m" + eventsPerCycle: 200 + randomness: 0.2 + - template: "good" + start: "now-6d-4h-44m" + end: "now-1d" + eventsPerCycle: 200 + randomness: 0.2 + - template: "bad" + start: "now-1d" + end: "now-1d+45m" + eventsPerCycle: 200 + randomness: 0.2 + - template: "good" + start: "now-1d+45m" + end: false + eventsPerCycle: 200 + randomness: 0.2 diff --git a/x-pack/packages/kbn-data-forge/example_config/full_example.yaml b/x-pack/packages/kbn-data-forge/example_config/full_example.yaml new file mode 100644 index 0000000000000..a7af3d5c7b8b6 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/full_example.yaml @@ -0,0 +1,41 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + host: "http://localhost:9200" + username: "elastic" + password: "changeme" + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: true + +# Define the indexing options +indexing: + dataset: "fake_stack" + interval: 6000 + eventsPerCycle: 1 + payloadSize: 10000 + concurrency: 5 + +# Define the schedule +schedule: + - template: "good" + start: "now-1h" + end: "now-15m" + - template: "bad" + start: "now-15m" + end: "now" + # This schedule will run indefinitely + - template: "good" + start: "now" + end: false + # This will add a 2 minute delay to the indexing every 5 mintes. + # Once the 2 minutes is up, the queue will back fill the events it collected + # during the delay. This only makes sense if `end` is `false` + delayInMinutes: 2 + delayEveryMinutes: 5 + diff --git a/x-pack/packages/kbn-data-forge/example_config/future_example.yaml b/x-pack/packages/kbn-data-forge/example_config/future_example.yaml new file mode 100644 index 0000000000000..c879ee34c3959 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/future_example.yaml @@ -0,0 +1,20 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_stack" + +schedule: + - template: "good" + start: "now" + end: "now+1h" + - template: "bad" + start: "now+1h" + end: "now+90m" + - template: "good" + start: "now+90m" + end: "now+2h" diff --git a/x-pack/packages/kbn-data-forge/example_config/good_to_bad_to_good.yaml b/x-pack/packages/kbn-data-forge/example_config/good_to_bad_to_good.yaml new file mode 100644 index 0000000000000..f460227af6bce --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/good_to_bad_to_good.yaml @@ -0,0 +1,20 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_stack" + +schedule: + - template: "good" + start: "now-2h" + end: "now-1h" + - template: "bad" + start: "now-1h" + end: "now-15m" + - template: "good" + start: "now-15m" + end: false diff --git a/x-pack/packages/kbn-data-forge/example_config/log_drop.yml b/x-pack/packages/kbn-data-forge/example_config/log_drop.yml new file mode 100644 index 0000000000000..a46956e2b7ddf --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/log_drop.yml @@ -0,0 +1,39 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +indexing: + eventsPerCycle: 100 + dataset: "fake_logs" + scenario: "log_drop" + +schedule: + # Start with good events at 100 rate + - template: "good" + start: "now-30m" + end: "now-5m" + eventsPerCycle: 50 + randomness: 0.1 + # Create a ramp-up of bad events + - template: "good" + start: "now-5m" + end: "now+5m" + eventsPerCycle: + start: 50 + end: 100 + method: "linear" + randomness: 0.1 + # Drop to very little good events + - template: "good" + start: "now+5m" + end: false + eventsPerCycle: 10 + randomness: 0.1 \ No newline at end of file diff --git a/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario0_logs.yaml b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario0_logs.yaml new file mode 100644 index 0000000000000..6a4ee8b184557 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario0_logs.yaml @@ -0,0 +1,29 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + host: "http://localhost:9200" + username: "elastic" + password: "changeme" + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_logs" + eventsPerCycle: 2000 + interval: 6000 + +# Define the schedule +schedule: + # This step send 2000 logs every 30 seconds + - template: "good" + eventsPerCycle: 2000 + start: "now-5m" + end: "now+5m" + diff --git a/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario1_spike_logs.yaml b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario1_spike_logs.yaml new file mode 100644 index 0000000000000..e12462aebe363 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario1_spike_logs.yaml @@ -0,0 +1,35 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + host: "http://localhost:9200" + username: "elastic" + password: "changeme" + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_logs" + eventsPerCycle: 2000 + interval: 6000 + +# Define the schedule +schedule: + # This step send 2000 logs every 30 seconds + - template: "good" + eventsPerCycle: 2000 + start: "now-2m" + end: "now+1m" + # This step send 6000 logs every 30 seconds + - template: "bad" + start: "now+1m" + end: "now+10m" + eventsPerCycle: 6000 + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario2_spike_logs_host.yaml b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario2_spike_logs_host.yaml new file mode 100644 index 0000000000000..bd52900d6fb5d --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario2_spike_logs_host.yaml @@ -0,0 +1,35 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + host: "http://localhost:9200" + username: "elastic" + password: "changeme" + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_logs" + eventsPerCycle: 2000 + interval: 6000 + concurrency: 4 + +# Define the schedule +schedule: + # This step send 2000 logs every 30 seconds for 4 hosts + - template: "good" + eventsPerCycle: 2000 + start: "now-2m" + end: "now+1m" + # This step send 8000 logs every 30 seconds for 4 hosts + - template: "bad" + start: "now+1m" + end: "now+5m" + eventsPerCycle: 8000 + randomness: 0.1 diff --git a/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario3_spike_errors.yaml b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario3_spike_errors.yaml new file mode 100644 index 0000000000000..6cfa6b7b76b39 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario3_spike_errors.yaml @@ -0,0 +1,33 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + host: "http://localhost:9200" + username: "elastic" + password: "changeme" + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_stack" + interval: 6000 + eventsPerCycle: 1 + payloadSize: 10000 + concurrency: 5 + +# Define the schedule +schedule: + # This step will send "good" events + - template: "good" + start: "now-2m" + end: "now+1m" + # This step will send "bad" events, with errors + - template: "bad" + start: "now+1m" + end: "now+10m" diff --git a/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario4_spike_errors_with_recovery.yaml b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario4_spike_errors_with_recovery.yaml new file mode 100644 index 0000000000000..5c7906889ed29 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario4_spike_errors_with_recovery.yaml @@ -0,0 +1,37 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + host: "http://localhost:9200" + username: "elastic" + password: "changeme" + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_stack" + interval: 6000 + eventsPerCycle: 1 + payloadSize: 10000 + concurrency: 5 + +# Define the schedule +schedule: + # This step will send "good" events + - template: "good" + start: "now-2m" + end: "now+1m" + # This step will send "bad" events, with errors + - template: "bad" + start: "now+1m" + end: "now+5m" + # Recover to good events + - template: "good" + start: "now+5m" + end: "now+10m" diff --git a/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario5_spike_logs_linear.yaml b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario5_spike_logs_linear.yaml new file mode 100644 index 0000000000000..2dd5c4283893e --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/log_spike_scenarios/scenario5_spike_logs_linear.yaml @@ -0,0 +1,40 @@ +--- +# Define the connection to Elasticsearch +elasticsearch: + installKibanaUser: false + +# Define the connection to Kibana +kibana: + host: "http://localhost:5601" + username: "elastic" + password: "changeme" + installAssets: false + +# Define the indexing options +indexing: + dataset: "fake_logs" + eventsPerCycle: 2000 + interval: 6000 + +# Define the schedule +schedule: + # This step send 2000 logs every 30 seconds + - template: "good" + eventsPerCycle: 2000 + start: "now-2m" + end: "now+1m" + - template: "good" + start: "now+1m" + end: "now+5m" + randomness: 0.1 + eventsPerCycle: + start: 2000 + end: 6000 + method: "linear" + # This step send 6000 logs every 30 seconds + - template: "bad" + start: "now+5m" + end: "now+10m" + eventsPerCycle: 6000 + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/metric_example.yaml b/x-pack/packages/kbn-data-forge/example_config/metric_example.yaml new file mode 100644 index 0000000000000..d4b10d52fcb55 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/metric_example.yaml @@ -0,0 +1,31 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_hosts" + eventsPerCycle: 1 + interval: 10000 + +schedule: + # Start with good events + - template: "good" + start: "now-2h" + end: false + eventsPerCycle: 1 + metrics: + - name: "system.cpu.user.pct" + method: "sine" + start: 1 + end: 4 + period: 2500 + randomness: 0.1 + - name: "system.cpu.system.pct" + method: "exp" + start: 1 + end: 4 + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/ramp_up_then_down.yaml b/x-pack/packages/kbn-data-forge/example_config/ramp_up_then_down.yaml new file mode 100644 index 0000000000000..a62b5064350f9 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/ramp_up_then_down.yaml @@ -0,0 +1,59 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + dataset: "fake_stack" + eventsPerCycle: 100 + +schedule: + # Ramp up + - template: "good" + start: "now" + end: "now+60m" + eventsPerCycle: + start: 100 + end: 1 + method: 'linear' + randomness: 0.1 + - template: "bad" + start: "now" + end: "now+60m" + eventsPerCycle: + start: 1 + end: 100 + method: 'linear' + randomness: 0.1 + # Level off + - template: "bad" + start: "now+60m" + end: "now+90m" + eventsPerCycle: 100 + randomness: 0.1 + # Ramp down + - template: "good" + start: "now+90m" + end: "now+150m" + eventsPerCycle: + start: 1 + end: 100 + method: 'linear' + randomness: 0.1 + - template: "bad" + start: "now+90m" + end: "now+150m" + eventsPerCycle: + start: 100 + end: 1 + method: 'linear' + randomness: 0.1 + # Back to normal + - template: "good" + start: "now+150m" + end: "now+210m" + eventsPerCycle: 100 + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario0_paralell_metrics_drop_step_change.yaml b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario0_paralell_metrics_drop_step_change.yaml new file mode 100644 index 0000000000000..7a153ad80fc28 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario0_paralell_metrics_drop_step_change.yaml @@ -0,0 +1,26 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_hosts" + scenario: "scenario0" + eventsPerCycle: 1 + interval: 10000 + +schedule: + # Start events at 100 rate + - template: "good" + start: "now-30m" + end: "now+2m" + eventsPerCycle: 100 + randomness: 0.1 + # Step change to events at 10 rate + - template: "good" + start: "now+2m" + end: "now+10m" + eventsPerCycle: 10 + diff --git a/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario1_paralell_metrics_increase_step_change.yaml b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario1_paralell_metrics_increase_step_change.yaml new file mode 100644 index 0000000000000..fb80dc2bc3362 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario1_paralell_metrics_increase_step_change.yaml @@ -0,0 +1,35 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_hosts" + scenario: "scenario1" + eventsPerCycle: 1 + interval: 10000 + +schedule: + # Start events at 100 rate + - template: "good" + start: "now-30m" + end: "now+2m" + eventsPerCycle: 50 + randomness: 0.1 + # Step change to events at 20 rate + - template: "good" + start: "now+2m" + end: "now+3m" + eventsPerCycle: + start: 50 + end: 200 + method: "linear" + randomness: 0.1 + - template: "good" + start: "now+3m" + end: "now+15m" + eventsPerCycle: 200 + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario2_divergent_metrics.yaml b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario2_divergent_metrics.yaml new file mode 100644 index 0000000000000..1cdff47bb2348 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario2_divergent_metrics.yaml @@ -0,0 +1,43 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_hosts" + scenario: "scenario2" + eventsPerCycle: 50 + interval: 10000 + +schedule: + # Start with good events + - template: "good" + start: "now-30m" + end: "now+2m" + randomness: 0.1 + - template: "good" + start: "now+2m" + end: "now+5m" + metrics: + - name: "system.cpu.user.pct" + method: "exp" + start: 10 + end: 2.5 + randomness: 0.1 + - name: "system.memory.actual.used.pct" + method: "exp" + start: 2.5 + end: 0.5 + randomness: 0.1 + - name: "system.filesystem.used.pct" + method: "linear" + start: 2.5 + end: 0.5 + randomness: 0.1 + - template: "good" + start: "now+5m" + end: "now+15m" + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario3_chained_metrics_change.yaml b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario3_chained_metrics_change.yaml new file mode 100644 index 0000000000000..687e718919cfb --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/related_events_metrics/scenario3_chained_metrics_change.yaml @@ -0,0 +1,59 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_hosts" + scenario: "scenario3" + eventsPerCycle: 50 + interval: 10000 + +schedule: + # Start with good events + - template: "good" + start: "now-30m" + end: "now+2m" + randomness: 0.1 + # Create a spike on a metric + - template: "good" + start: "now+1m" + end: "now+2m" + randomness: 0.1 + metrics: + - name: "system.load.1" + method: "linear" + start: 2.5 + end: 30 + randomness: 0.1 + - template: "good" + start: "now+2m" + end: "now+3m" + randomness: 0.1 + metrics: + - name: "system.cpu.user.pct" + method: "exp" + start: 2.5 + end: 10 + randomness: 0.1 + - template: "good" + start: "now+3m" + end: "now+5m" + metrics: + - name: "system.memory.actual.used.pct" + method: "exp" + start: 2.5 + end: 0.5 + randomness: 0.1 + - name: "system.filesystem.used.pct" + method: "linear" + start: 2.5 + end: 0.5 + randomness: 0.1 + - template: "good" + start: "now+5m" + end: "now+15m" + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count.yaml b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count.yaml new file mode 100644 index 0000000000000..65f84997f6815 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count.yaml @@ -0,0 +1,27 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + dataset: "fake_hosts" + scenario: "custom_threshold_log_count" + eventsPerCycle: 50 + concurrency: 5 + +# Conditions: count logs < 100 in the last minute +schedule: + # Start events at 100 rate + - template: "good" + start: "now-30m" + end: "now-1m" + eventsPerCycle: 100 + randomness: 0.1 + # Step change to events at 10 rate + - template: "good" + start: "now-1m" + end: "now+20m" + eventsPerCycle: 10 + diff --git a/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_groupby.yaml b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_groupby.yaml new file mode 100644 index 0000000000000..101ddde3e93d5 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_groupby.yaml @@ -0,0 +1,27 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + dataset: "fake_hosts" + scenario: "custom_threshold_log_count_groupby" + eventsPerCycle: 50 + concurrency: 5 + +# Conditions: count logs < 40 in the last minute, grouped by event.dataset +schedule: + # Start events at 100 rate + - template: "good" + start: "now-30m" + end: "now-1m" + eventsPerCycle: 100 + randomness: 0.1 + # Step change to events at 10 rate + - template: "good" + start: "now-1m" + end: "now+20m" + eventsPerCycle: 10 + diff --git a/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_nodata.yaml b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_nodata.yaml new file mode 100644 index 0000000000000..7023f24bb3def --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_log_count_nodata.yaml @@ -0,0 +1,23 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + dataset: "fake_hosts" + scenario: "custom_threshold_log_count_nodata" + eventsPerCycle: 50 + concurrency: 5 + +# Conditions: count logs < 5 in the last minute (will trigger no data after 1 min) +schedule: + # Start events at 10 rate + - template: "good" + start: "now-30m" + end: "now-1m" + eventsPerCycle: 10 + randomness: 0.1 + # Stop data + diff --git a/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg.yaml b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg.yaml new file mode 100644 index 0000000000000..d8cb165e78c70 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg.yaml @@ -0,0 +1,44 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + dataset: "fake_hosts" + scenario: "custom_threshold_metric_avg" + eventsPerCycle: 50 + +# Conditions: avg. system.cpu.user.pct > 80 in the last minute +schedule: + # Start with avg. system.cpu.user.pct = 2.5 + - template: "good" + start: "now-30m" + end: "now-3m" + randomness: 0.1 + # Transition to avg. system.cpu.user.pct = 90 + - template: "good" + start: "now-3m" + end: "now-2m" + metrics: + - name: "system.cpu.user.pct" + method: "linear" + start: 2.5 + end: 90 + randomness: 0.1 + - template: "good" + start: "now-2m" + end: "now+13m" + metrics: + - name: "system.cpu.user.pct" + method: "linear" + start: 90 + end: 90 + randomness: 0.05 + # Go back to avg. system.cpu.user.pct = 2.5 + - template: "good" + start: "now+13m" + end: "now+25m" + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_groupby.yaml b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_groupby.yaml new file mode 100644 index 0000000000000..280a85dd1437c --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_groupby.yaml @@ -0,0 +1,45 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + dataset: "fake_hosts" + scenario: "custom_threshold_metric_avg_groupby" + eventsPerCycle: 5 + interval: 5000 + +# Conditions: avg. system.cpu.user.pct > 80 in the last 5 minutes, grouped by host.name +schedule: + # Start with avg. system.cpu.user.pct = 2.5 + - template: "good" + start: "now-30m" + end: "now-6m" + randomness: 0.1 + # Transition to avg. system.cpu.user.pct = 90 + - template: "good" + start: "now-6m" + end: "now-5m" + metrics: + - name: "system.cpu.user.pct" + method: "linear" + start: 2.5 + end: 90 + randomness: 0.1 + - template: "good" + start: "now-5m" + end: "now+23m" + metrics: + - name: "system.cpu.user.pct" + method: "linear" + start: 90 + end: 90 + randomness: 0.05 + # Go back to avg. system.cpu.user.pct = 2.5 + - template: "good" + start: "now+23m" + end: "now+45m" + randomness: 0.1 + diff --git a/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_nodata.yaml b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_nodata.yaml new file mode 100644 index 0000000000000..ed964fe97f347 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/rule_tests/custom_threshold_metric_avg_nodata.yaml @@ -0,0 +1,20 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + dataset: "fake_hosts" + scenario: "custom_threshold_metric_avg_nodata" + eventsPerCycle: 50 + +# Conditions: avg. system.cpu.user.pct < 1 in the last minute (will trigger no data after 2 min) +schedule: + # Start with avg. system.cpu.user.pct = 2.5 + - template: "good" + start: "now-30m" + end: "now-1m" + randomness: 0.1 + # Stop data diff --git a/x-pack/packages/kbn-data-forge/example_config/rule_tests/slo_burn_rate.yaml b/x-pack/packages/kbn-data-forge/example_config/rule_tests/slo_burn_rate.yaml new file mode 100644 index 0000000000000..8bf7d058760e1 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/rule_tests/slo_burn_rate.yaml @@ -0,0 +1,22 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: false + +indexing: + dataset: "fake_stack" + interval: 3600000 + +schedule: + # Start with good events + - template: "good" + start: "now-7d" + end: "now-45m" + - template: "bad" + start: "now-2h" + end: "now+25m" + randomness: 0.2 + interval: 60000 + diff --git a/x-pack/packages/kbn-data-forge/example_config/transition_example.yaml b/x-pack/packages/kbn-data-forge/example_config/transition_example.yaml new file mode 100644 index 0000000000000..b161d019bef5d --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/transition_example.yaml @@ -0,0 +1,45 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + eventsPerCycle: 1000 + dataset: "fake_logs" + +schedule: + - template: "good" + start: "now-90m" + end: "now-75m" + eventsPerCycle: 1000 + randomness: 0.1 + - template: "bad" + start: "now-75m" + end: "now-60m" + randomness: 0.1 + eventsPerCycle: + start: 1000 + end: 5000 + method: "exp" + - template: "bad" + start: "now-60m" + end: "now-45m" + eventsPerCycle: 5000 + randomness: 0.1 + - template: "good" + start: "now-45m" + end: "now-30m" + eventsPerCycle: + start: 5000 + end: 1000 + method: "exp" + randomness: 0.1 + - template: "good" + start: "now-30m" + end: false + eventsPerCycle: 1000 + randomness: 0.1 + + diff --git a/x-pack/packages/kbn-data-forge/example_config/transitioning_templates_example.yaml b/x-pack/packages/kbn-data-forge/example_config/transitioning_templates_example.yaml new file mode 100644 index 0000000000000..f5d3cbb50fef9 --- /dev/null +++ b/x-pack/packages/kbn-data-forge/example_config/transitioning_templates_example.yaml @@ -0,0 +1,66 @@ +--- +elasticsearch: + installKibanaUser: false + +kibana: + installAssets: true + +indexing: + eventsPerCycle: 100 + dataset: "fake_stack" + +schedule: + # Start with good events + - template: "good" + start: "now-90m" + end: "now-45m" + eventsPerCycle: 100 + randomness: 0.1 + # Transition from good to bad by setting up two schedules witht he same + # time frames but different templates to transition between 0 to 100 and vice + # versa + - template: "good" + start: "now-45m" + end: "now-35m" + eventsPerCycle: + start: 100 + end: 1 + method: "linear" + randomness: 0.1 + - template: "bad" + start: "now-45m" + end: "now-35m" + eventsPerCycle: + start: 1 + end: 100 + method: "linear" + randomness: 0.1 + # Bad for 10 minutes + - template: "bad" + start: "now-35m" + end: "now-25m" + eventsPerCycle: 100 + randomness: 0.1 + # Transition back from bad to good + - template: "good" + start: "now-25m" + end: "now-15m" + eventsPerCycle: + start: 1 + end: 100 + method: "linear" + randomness: 0.1 + - template: "bad" + start: "now-25m" + end: "now-15m" + eventsPerCycle: + start: 100 + end: 1 + method: "linear" + randomness: 0.1 + # continue with good + - template: "good" + start: "now-15m" + end: false + eventsPerCycle: 100 + randomness: 0.1