From 3a299427add1260dcb8d9ec2b443a73a706d2ca2 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Tue, 1 Sep 2020 18:16:24 -0400 Subject: [PATCH 01/10] adds metrics ml integration --- .../modules/metrics_ui_hosts/logo.json | 3 + .../modules/metrics_ui_hosts/manifest.json | 59 +++++++++++++++++ .../ml/datafeed_hosts_cpu_usage.json | 66 +++++++++++++++++++ .../ml/datafeed_hosts_memory_usage.json | 13 ++++ .../ml/datafeed_hosts_network_in.json | 37 +++++++++++ .../ml/datafeed_hosts_network_out.json | 37 +++++++++++ .../metrics_ui_hosts/ml/hosts_cpu_usage.json | 41 ++++++++++++ .../ml/hosts_memory_usage.json | 41 ++++++++++++ .../metrics_ui_hosts/ml/hosts_network_in.json | 34 ++++++++++ .../ml/hosts_network_out.json | 31 +++++++++ .../modules/metrics_ui_k8s/logo.json | 3 + .../modules/metrics_ui_k8s/manifest.json | 56 ++++++++++++++++ .../modules/metrics_ui_k8s/ml/cpu_usage.json | 44 +++++++++++++ .../metrics_ui_k8s/ml/datafeed_cpu_usage.json | 14 ++++ .../ml/datafeed_memory_usage.json | 14 ++++ .../ml/datafeed_network_in.json | 41 ++++++++++++ .../ml/datafeed_network_out.json | 41 ++++++++++++ .../metrics_ui_k8s/ml/memory_usage.json | 44 +++++++++++++ .../modules/metrics_ui_k8s/ml/network_in.json | 36 ++++++++++ .../metrics_ui_k8s/ml/network_out.json | 36 ++++++++++ .../apis/ml/modules/get_module.ts | 2 + 21 files changed, 693 insertions(+) create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/cpu_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_cpu_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_memory_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_in.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_out.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/memory_usage.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_in.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_out.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json new file mode 100644 index 0000000000000..2e57038bbc639 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "metricsApp" +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json new file mode 100644 index 0000000000000..8338a084ee688 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json @@ -0,0 +1,59 @@ +{ + "id": "metrics_hosts", + "title": "Metrics Hosts", + "description": "", + "type": "Metricbeat Data", + "logoFile": "logo.json", + "defaultIndexPattern": "metricbeat-*", + "query": { + "bool": { + "must": [ + {"exists": {"field": "system"}} + ], + "must_not": [ + {"bool": {"filter": [{"exists":{"field": "kubernetes"}}]}}, + {"bool": {"filter": [{"exists":{"field": "docker"}}]}} + ] + } + }, + "jobs": [ + { + "id": "hosts_cpu_usage", + "file": "hosts_cpu_usage.json" + }, + { + "id": "hosts_memory_usage", + "file": "hosts_memory_usage.json" + }, + { + "id": "hosts_network_in", + "file": "hosts_network_in.json" + }, + { + "id": "hosts_network_out", + "file": "hosts_network_out.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-hosts_cpu_usage", + "file": "datafeed_hosts_cpu_usage.json", + "job_id": "hosts_cpu_usage" + }, + { + "id": "datafeed-hosts_memory_usage", + "file": "datafeed_hosts_memory_usage.json", + "job_id": "hosts_memory_usage" + }, + { + "id": "datafeed-hosts_network_in", + "file": "datafeed_hosts_network_in.json", + "job_id": "hosts_network_in" + }, + { + "id": "datafeed-hosts_network_out", + "file": "datafeed_hosts_network_out.json", + "job_id": "hosts_network_out" + } + ] +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json new file mode 100644 index 0000000000000..784a5d1059109 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json @@ -0,0 +1,66 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "system.cpu" + } + } + ] + } + }, + "aggs": { + "host.name": { + "terms": { + "field": "host.name", + "size": 1000 + }, + "aggs": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "1m" + }, + "aggs": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "cpu_user": { + "avg": { + "field": "system.cpu.user.pct" + } + }, + "cpu_system": { + "avg": { + "field": "system.cpu.system.pct" + } + }, + "cpu_cores": { + "max": { + "field": "system.cpu.cores" + } + }, + "cpu": { + "bucket_script": { + "buckets_path": { + "user": "cpu_user", + "system": "cpu_system", + "cores": "cpu_cores" + }, + "script": "(params.user + params.system) / params.cores", + "gap_policy": "skip" + } + } + } + } + } + } + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json new file mode 100644 index 0000000000000..abd3249e91385 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + {"exists": {"field": "system.memory"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json new file mode 100644 index 0000000000000..b6bb8e25334fc --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json @@ -0,0 +1,37 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + {"exists": {"field": "system.network"}} + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "1000s" + }, + "aggregations": { + "host.name": {"terms": {"field": "host.name"}, + "aggregations": { + "buckets": { + "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, + "aggregations": { + "@timestamp": {"max": {"field": "@timestamp"}}, + "bytes_in_max": {"max": {"field": "system.network.in.bytes"}}, + "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}}, + "positive_only":{ + "bucket_script": { + "buckets_path": {"in_derivative": "bytes_in_derivative.value"}, + "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json new file mode 100644 index 0000000000000..6494b2c7bf915 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json @@ -0,0 +1,37 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + {"exists": {"field": "system.network"}} + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "1000s" + }, + "aggregations": { + "host.name": {"terms": {"field": "host.name"}, + "aggregations": { + "buckets": { + "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, + "aggregations": { + "@timestamp": {"max": {"field": "@timestamp"}}, + "bytes_out_max": {"max": {"field": "system.network.out.bytes"}}, + "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}}, + "positive_only":{ + "bucket_script": { + "buckets_path": {"in_derivative": "bytes_out_derivative.value"}, + "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json new file mode 100644 index 0000000000000..9ccba4b45e977 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json @@ -0,0 +1,41 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "hosts", + "metrics" + ], + "description": "", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max('cpu') partitionfield='host.name'", + "function": "max", + "field_name": "cpu", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.1 + } + ] + } + ] + } + ], + "influencers": [ + "host.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-hosts" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json new file mode 100644 index 0000000000000..a7fd32b09872b --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json @@ -0,0 +1,41 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "hosts", + "metrics" + ], + "description": "", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max(\"system.memory.actual.used.pct\") partitionfield=\"host.name\"", + "function": "max", + "field_name": "system.memory.actual.used.pct", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.1 + } + ] + } + ] + } + ], + "influencers": [ + "host.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-hosts" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json new file mode 100644 index 0000000000000..d0604828cddd8 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json @@ -0,0 +1,34 @@ +{ + "job_type": "anomaly_detector", + "description": "", + "groups": [ + "hosts", + "metrics" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max(bytes_in_derivative)", + "function": "max", + "field_name": "bytes_in_derivative" + } + ], + "influencers": [ + "host.name" + ], + "summary_count_field_name": "doc_count" + }, + "model_plot_config": { + "enabled": "true" + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-hosts" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json new file mode 100644 index 0000000000000..62a11e4f59b36 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json @@ -0,0 +1,31 @@ +{ + "job_type": "anomaly_detector", + "description": "", + "groups": [ + "hosts", + "metrics" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max(bytes_out_derivative)", + "function": "max", + "field_name": "bytes_out_derivative" + } + ], + "influencers": [ + "host.name" + ], + "summary_count_field_name": "doc_count" + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-hosts" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json new file mode 100644 index 0000000000000..63105a28c0ab1 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "metricsApp" +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json new file mode 100644 index 0000000000000..c8da208fb4a8a --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json @@ -0,0 +1,56 @@ +{ + "id": "metrics_k8s", + "title": "Metrics Kubernetes", + "description": "", + "type": "Metricbeat Data", + "logoFile": "logo.json", + "defaultIndexPattern": "metricbeat-*", + "query": { + "bool": { + "filter": [ + {"exists": {"field": "kubernetes"}}, + {"term": {"event.module": "kubernetes"}} + ] + } + }, + "jobs": [ + { + "id": "cpu_usage", + "file": "cpu_usage.json" + }, + { + "id": "memory_usage", + "file": "memory_usage.json" + }, + { + "id": "network_in", + "file": "network_in.json" + }, + { + "id": "network_out", + "file": "network_out.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-cpu_usage", + "file": "datafeed_cpu_usage.json", + "job_id": "cpu_usage" + }, + { + "id": "datafeed-memory_usage", + "file": "datafeed_memory_usage.json", + "job_id": "memory_usage" + }, + { + "id": "datafeed-network_in", + "file": "datafeed_network_in.json", + "job_id": "network_in" + }, + { + "id": "datafeed-network_out", + "file": "datafeed_network_out.json", + "job_id": "network_out" + } + ] + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/cpu_usage.json new file mode 100644 index 0000000000000..9fe7874811c04 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/cpu_usage.json @@ -0,0 +1,44 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "k8s", + "metrics" + ], + "description": "", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max('kubernetes.pod.cpu.usage.node.pct') partitionfield='kubernetes.namespace'", + "function": "max", + "field_name": "kubernetes.pod.cpu.usage.node.pct", + "partition_field_name": "kubernetes.namespace", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.1 + } + ] + } + ] + } + ], + "influencers": [ + "kubernetes.namespace", + "kubernetes.node.name", + "kubernetes.pod.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-k8s" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_cpu_usage.json new file mode 100644 index 0000000000000..fd74174795e09 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_cpu_usage.json @@ -0,0 +1,14 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + {"exists": {"field": "kubernetes.pod.uid"}}, + {"exists": {"field": "kubernetes.pod.cpu"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_memory_usage.json new file mode 100644 index 0000000000000..161867d615fae --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_memory_usage.json @@ -0,0 +1,14 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + {"exists": {"field": "kubernetes.pod.uid"}}, + {"exists": {"field": "kubernetes.pod.memory"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_in.json new file mode 100644 index 0000000000000..d136025bc1c8e --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_in.json @@ -0,0 +1,41 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + {"exists": {"field": "kubernetes.pod.network"}} + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "1000s" + }, + "aggregations": { + "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace"}, + "aggregations": { + "kubernetes.pod.name": {"terms": {"field": "kubernetes.pod.name"}, + "aggregations": { + "buckets": { + "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, + "aggregations": { + "@timestamp": {"max": {"field": "@timestamp"}}, + "bytes_in_max": {"max": {"field": "kubernetes.pod.network.rx.bytes"}}, + "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}}, + "positive_only":{ + "bucket_script": { + "buckets_path": {"in_derivative": "bytes_in_derivative.value"}, + "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" + } + } + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_out.json new file mode 100644 index 0000000000000..ec599206030d5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_out.json @@ -0,0 +1,41 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": [ + {"exists": {"field": "kubernetes.pod.network"}} + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "1000s" + }, + "aggregations": { + "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace"}, + "aggregations": { + "kubernetes.pod.name": {"terms": {"field": "kubernetes.pod.name"}, + "aggregations": { + "buckets": { + "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, + "aggregations": { + "@timestamp": {"max": {"field": "@timestamp"}}, + "bytes_out_max": {"max": {"field": "kubernetes.pod.network.tx.bytes"}}, + "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}}, + "positive_only":{ + "bucket_script": { + "buckets_path": {"pos_derivative": "bytes_out_derivative.value"}, + "script": "params.pos_derivative > 0.0 ? params.pos_derivative : 0.0" + } + } + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/memory_usage.json new file mode 100644 index 0000000000000..cb5f8127b780b --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/memory_usage.json @@ -0,0 +1,44 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "k8s", + "metrics" + ], + "description": "", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max('kubernetes.pod.memory.usage.node.pct') partitionfield='kubernetes.namespace'", + "function": "max", + "field_name": "kubernetes.pod.memory.usage.node.pct", + "partition_field_name": "kubernetes.namespace", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.1 + } + ] + } + ] + } + ], + "influencers": [ + "kubernetes.namespace", + "kubernetes.node.name", + "kubernetes.pod.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-k8s" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_in.json new file mode 100644 index 0000000000000..a48d3b6e0608a --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_in.json @@ -0,0 +1,36 @@ +{ + "job_type": "anomaly_detector", + "description": "", + "groups": [ + "k8s", + "metrics" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max(bytes_in_derivative)", + "function": "max", + "field_name": "bytes_in_derivative", + "partition_field_name": "kubernetes.namespace" + } + ], + "influencers": [ + "kubernetes.namespace", + "kubernetes.pod.name" + ], + "summary_count_field_name": "doc_count" + }, + "model_plot_config": { + "enabled": "true" + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-k8s" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_out.json new file mode 100644 index 0000000000000..cf4507f9f5d8f --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_out.json @@ -0,0 +1,36 @@ +{ + "job_type": "anomaly_detector", + "description": "", + "groups": [ + "k8s", + "metrics" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max(bytes_out_derivative)", + "function": "max", + "field_name": "bytes_out_derivative", + "partition_field_name": "kubernetes.namespace" + } + ], + "influencers": [ + "kubernetes.namespace", + "kubernetes.pod.name" + ], + "summary_count_field_name": "doc_count" + }, + "model_plot_config": { + "enabled": "true" + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-k8s" + } +} diff --git a/x-pack/test/api_integration/apis/ml/modules/get_module.ts b/x-pack/test/api_integration/apis/ml/modules/get_module.ts index a3d060bb1faca..6c7cb8bf4dce0 100644 --- a/x-pack/test/api_integration/apis/ml/modules/get_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/get_module.ts @@ -20,6 +20,8 @@ const moduleIds = [ 'logs_ui_analysis', 'logs_ui_categories', 'metricbeat_system_ecs', + 'metrics_ui_hosts', + 'metrics_ui_k8s', 'nginx_ecs', 'sample_data_ecommerce', 'sample_data_weblogs', From 1d5e0d1dfe87bd59eb1c0bd413bf9f336fd028cc Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Tue, 8 Sep 2020 18:28:33 -0400 Subject: [PATCH 02/10] renames jobs, updates datafeeds --- .../ml/datafeed_hosts_network_out.json | 2 +- .../metrics_ui_hosts/ml/hosts_cpu_usage.json | 6 ++- .../ml/hosts_memory_usage.json | 5 ++- .../modules/metrics_ui_k8s/manifest.json | 40 +++++++++---------- ...usage.json => datafeed_k8s_cpu_usage.json} | 0 ...ge.json => datafeed_k8s_memory_usage.json} | 0 ...k_in.json => datafeed_k8s_network_in.json} | 0 ...out.json => datafeed_k8s_network_out.json} | 0 .../ml/{cpu_usage.json => k8s_cpu_usage.json} | 3 ++ ...emory_usage.json => k8s_memory_usage.json} | 3 ++ .../{network_in.json => k8s_network_in.json} | 0 ...{network_out.json => k8s_network_out.json} | 0 12 files changed, 36 insertions(+), 23 deletions(-) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{datafeed_cpu_usage.json => datafeed_k8s_cpu_usage.json} (100%) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{datafeed_memory_usage.json => datafeed_k8s_memory_usage.json} (100%) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{datafeed_network_in.json => datafeed_k8s_network_in.json} (100%) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{datafeed_network_out.json => datafeed_k8s_network_out.json} (100%) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{cpu_usage.json => k8s_cpu_usage.json} (94%) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{memory_usage.json => k8s_memory_usage.json} (94%) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{network_in.json => k8s_network_in.json} (100%) rename x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/{network_out.json => k8s_network_out.json} (100%) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json index 6494b2c7bf915..42a8c99ac0fa6 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json @@ -25,7 +25,7 @@ "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}}, "positive_only":{ "bucket_script": { - "buckets_path": {"in_derivative": "bytes_out_derivative.value"}, + "buckets_path": {"out_derivative": "bytes_out_derivative.value"}, "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0" } } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json index 9ccba4b45e977..b4f2cb03ce83d 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json @@ -7,9 +7,10 @@ "description": "", "analysis_config": { "bucket_span": "15m", + "summary_count_field_name": "doc_count", "detectors": [ { - "detector_description": "max('cpu') partitionfield='host.name'", + "detector_description": "max('cpu')", "function": "max", "field_name": "cpu", "custom_rules": [ @@ -32,6 +33,9 @@ "host.name" ] }, + "data_description": { + "time_field": "@timestamp" + }, "analysis_limits": { "model_memory_limit": "64mb" }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json index a7fd32b09872b..923de0fa0104c 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json @@ -9,7 +9,7 @@ "bucket_span": "15m", "detectors": [ { - "detector_description": "max(\"system.memory.actual.used.pct\") partitionfield=\"host.name\"", + "detector_description": "max('system.memory.actual.used.pct')", "function": "max", "field_name": "system.memory.actual.used.pct", "custom_rules": [ @@ -32,6 +32,9 @@ "host.name" ] }, + "data_description": { + "time_field": "@timestamp" + }, "analysis_limits": { "model_memory_limit": "64mb" }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json index c8da208fb4a8a..8d059741df368 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json @@ -15,42 +15,42 @@ }, "jobs": [ { - "id": "cpu_usage", - "file": "cpu_usage.json" + "id": "k8s_cpu_usage", + "file": "k8s_cpu_usage.json" }, { - "id": "memory_usage", - "file": "memory_usage.json" + "id": "k8s_memory_usage", + "file": "k8s_memory_usage.json" }, { - "id": "network_in", - "file": "network_in.json" + "id": "k8s_network_in", + "file": "k8s_network_in.json" }, { - "id": "network_out", - "file": "network_out.json" + "id": "k8s_network_out", + "file": "k8s_network_out.json" } ], "datafeeds": [ { - "id": "datafeed-cpu_usage", - "file": "datafeed_cpu_usage.json", - "job_id": "cpu_usage" + "id": "datafeed-k8s_cpu_usage", + "file": "datafeed_k8s_cpu_usage.json", + "job_id": "k8s_cpu_usage" }, { - "id": "datafeed-memory_usage", - "file": "datafeed_memory_usage.json", - "job_id": "memory_usage" + "id": "datafeed-k8s_memory_usage", + "file": "datafeed_k8s_memory_usage.json", + "job_id": "k8s_memory_usage" }, { - "id": "datafeed-network_in", - "file": "datafeed_network_in.json", - "job_id": "network_in" + "id": "datafeed-k8s_network_in", + "file": "datafeed_k8s_network_in.json", + "job_id": "k8s_network_in" }, { - "id": "datafeed-network_out", - "file": "datafeed_network_out.json", - "job_id": "network_out" + "id": "datafeed-k8s_network_out", + "file": "datafeed_k8s_network_out.json", + "job_id": "k8s_network_out" } ] } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json similarity index 100% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_cpu_usage.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json similarity index 100% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_memory_usage.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json similarity index 100% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_in.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json similarity index 100% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_network_out.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json similarity index 94% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/cpu_usage.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json index 9fe7874811c04..0dcc8762a5aed 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json @@ -35,6 +35,9 @@ "kubernetes.pod.name" ] }, + "data_description": { + "time_field": "@timestamp" + }, "analysis_limits": { "model_memory_limit": "64mb" }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json similarity index 94% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/memory_usage.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json index cb5f8127b780b..abed5fa7cce56 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json @@ -35,6 +35,9 @@ "kubernetes.pod.name" ] }, + "data_description": { + "time_field": "@timestamp" + }, "analysis_limits": { "model_memory_limit": "64mb" }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json similarity index 100% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_in.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json similarity index 100% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/network_out.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json From 1a6198d212a083c417072a28d0d5b43658ffec2d Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Thu, 10 Sep 2020 17:12:23 -0400 Subject: [PATCH 03/10] adds allow_no_indices: true for datafeeds --- .../modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json | 3 +++ .../metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json | 3 +++ .../modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json | 3 +++ .../metrics_ui_hosts/ml/datafeed_hosts_network_out.json | 3 +++ .../modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json | 3 +++ .../modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json | 3 +++ .../modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json | 3 +++ .../modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json | 3 +++ 8 files changed, 24 insertions(+) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json index 784a5d1059109..3e0924041f4cb 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json index abd3249e91385..db883a6ce36f9 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json index b6bb8e25334fc..671d0ce185ad1 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json index 42a8c99ac0fa6..e0435e99bbac1 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json index fd74174795e09..8b9f7dafe2dda 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json index 161867d615fae..14590f743528e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json index d136025bc1c8e..cd7b3990465cb 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json index ec599206030d5..d719f91f0e9b0 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json @@ -3,6 +3,9 @@ "indices": [ "INDEX_PATTERN_NAME" ], + "indices_options": { + "allow_no_indices": true + }, "query": { "bool": { "must": [ From 3418147e6cafacb89f92adbdf1c4f0eb7ee34b18 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Thu, 10 Sep 2020 17:24:05 -0400 Subject: [PATCH 04/10] updates module ids in manifest --- .../data_recognizer/modules/metrics_ui_hosts/manifest.json | 2 +- .../models/data_recognizer/modules/metrics_ui_k8s/manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json index 8338a084ee688..203d64a960e47 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json @@ -1,5 +1,5 @@ { - "id": "metrics_hosts", + "id": "metrics_ui_hosts", "title": "Metrics Hosts", "description": "", "type": "Metricbeat Data", diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json index 8d059741df368..2b2e3adc779ef 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json @@ -1,5 +1,5 @@ { - "id": "metrics_k8s", + "id": "metrics_ui_k8s", "title": "Metrics Kubernetes", "description": "", "type": "Metricbeat Data", From bcd4620c719774db58fee997cf15e99d74423e42 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Tue, 15 Sep 2020 11:13:07 -0400 Subject: [PATCH 05/10] adds custom urls --- .../modules/metrics_ui_hosts/ml/hosts_cpu_usage.json | 8 +++++++- .../metrics_ui_hosts/ml/hosts_memory_usage.json | 8 +++++++- .../modules/metrics_ui_hosts/ml/hosts_network_in.json | 8 +++++++- .../modules/metrics_ui_hosts/ml/hosts_network_out.json | 8 +++++++- .../metrics_ui_k8s/ml/datafeed_k8s_network_in.json | 2 +- .../metrics_ui_k8s/ml/datafeed_k8s_network_out.json | 2 +- .../modules/metrics_ui_k8s/ml/k8s_cpu_usage.json | 10 ++++++++-- .../modules/metrics_ui_k8s/ml/k8s_memory_usage.json | 10 ++++++++-- .../modules/metrics_ui_k8s/ml/k8s_network_in.json | 10 ++++++++-- .../modules/metrics_ui_k8s/ml/k8s_network_out.json | 10 ++++++++-- 10 files changed, 62 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json index b4f2cb03ce83d..2c1f34159cd87 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json @@ -40,6 +40,12 @@ "model_memory_limit": "64mb" }, "custom_settings": { - "created_by": "ml-module-metrics-hosts" + "created_by": "ml-module-metrics-ui-hosts", + "custom_urls": [ + { + "url_name": "Host Metrics", + "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json index 923de0fa0104c..53667e3732c3a 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json @@ -39,6 +39,12 @@ "model_memory_limit": "64mb" }, "custom_settings": { - "created_by": "ml-module-metrics-hosts" + "created_by": "ml-module-metrics-ui-hosts", + "custom_urls": [ + { + "url_name": "Host Metrics", + "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json index d0604828cddd8..078221686f703 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json @@ -29,6 +29,12 @@ "model_memory_limit": "32mb" }, "custom_settings": { - "created_by": "ml-module-metrics-hosts" + "created_by": "ml-module-metrics-ui-hosts", + "custom_urls": [ + { + "url_name": "Host Metrics", + "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json index 62a11e4f59b36..5cd0c77a54125 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json @@ -26,6 +26,12 @@ "model_memory_limit": "32mb" }, "custom_settings": { - "created_by": "ml-module-metrics-hosts" + "created_by": "ml-module-metrics-ui-hosts", + "custom_urls": [ + { + "url_name": "Host Metrics", + "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json index cd7b3990465cb..059184bc0ab12 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json @@ -20,7 +20,7 @@ "aggregations": { "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace"}, "aggregations": { - "kubernetes.pod.name": {"terms": {"field": "kubernetes.pod.name"}, + "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid"}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json index d719f91f0e9b0..35ae4201eadad 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json @@ -20,7 +20,7 @@ "aggregations": { "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace"}, "aggregations": { - "kubernetes.pod.name": {"terms": {"field": "kubernetes.pod.name"}, + "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid"}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json index 0dcc8762a5aed..19f06920303a6 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json @@ -32,7 +32,7 @@ "influencers": [ "kubernetes.namespace", "kubernetes.node.name", - "kubernetes.pod.name" + "kubernetes.pod.uid" ] }, "data_description": { @@ -42,6 +42,12 @@ "model_memory_limit": "64mb" }, "custom_settings": { - "created_by": "ml-module-metrics-k8s" + "created_by": "ml-module-metrics-ui-k8s", + "custom_urls": [ + { + "url_name": "Pod Metrics", + "url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json index abed5fa7cce56..15720a823e2a5 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json @@ -32,7 +32,7 @@ "influencers": [ "kubernetes.namespace", "kubernetes.node.name", - "kubernetes.pod.name" + "kubernetes.pod.uid" ] }, "data_description": { @@ -42,6 +42,12 @@ "model_memory_limit": "64mb" }, "custom_settings": { - "created_by": "ml-module-metrics-k8s" + "created_by": "ml-module-metrics-ui-k8s", + "custom_urls": [ + { + "url_name": "Pod Metrics", + "url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json index a48d3b6e0608a..3b78270bc0ac4 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json @@ -17,7 +17,7 @@ ], "influencers": [ "kubernetes.namespace", - "kubernetes.pod.name" + "kubernetes.pod.uid" ], "summary_count_field_name": "doc_count" }, @@ -31,6 +31,12 @@ "model_memory_limit": "32mb" }, "custom_settings": { - "created_by": "ml-module-metrics-k8s" + "created_by": "ml-module-metrics-ui-k8s", + "custom_urls": [ + { + "url_name": "Pod Metrics", + "url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json index cf4507f9f5d8f..4437f04c32c01 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json @@ -17,7 +17,7 @@ ], "influencers": [ "kubernetes.namespace", - "kubernetes.pod.name" + "kubernetes.pod.uid" ], "summary_count_field_name": "doc_count" }, @@ -31,6 +31,12 @@ "model_memory_limit": "32mb" }, "custom_settings": { - "created_by": "ml-module-metrics-k8s" + "created_by": "ml-module-metrics-ui-k8s", + "custom_urls": [ + { + "url_name": "Pod Metrics", + "url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] } } From edb65d329c1d7558297271229bc07006a65eb144 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Tue, 15 Sep 2020 13:23:33 -0400 Subject: [PATCH 06/10] adds module and individual job descriptions --- .../data_recognizer/modules/metrics_ui_hosts/manifest.json | 2 +- .../modules/metrics_ui_hosts/ml/hosts_cpu_usage.json | 2 +- .../modules/metrics_ui_hosts/ml/hosts_memory_usage.json | 2 +- .../modules/metrics_ui_hosts/ml/hosts_network_in.json | 2 +- .../modules/metrics_ui_hosts/ml/hosts_network_out.json | 2 +- .../models/data_recognizer/modules/metrics_ui_k8s/manifest.json | 2 +- .../modules/metrics_ui_k8s/ml/k8s_cpu_usage.json | 2 +- .../modules/metrics_ui_k8s/ml/k8s_memory_usage.json | 2 +- .../modules/metrics_ui_k8s/ml/k8s_network_in.json | 2 +- .../modules/metrics_ui_k8s/ml/k8s_network_out.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json index 203d64a960e47..e36ea2bdc2a17 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json @@ -1,7 +1,7 @@ { "id": "metrics_ui_hosts", "title": "Metrics Hosts", - "description": "", + "description": "Detect anomalous memory, cpu, and network behavior on hosts.", "type": "Metricbeat Data", "logoFile": "logo.json", "defaultIndexPattern": "metricbeat-*", diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json index 2c1f34159cd87..7a598f7502ea6 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json @@ -4,7 +4,7 @@ "hosts", "metrics" ], - "description": "", + "description": "Metrics: Hosts - Identify unusual spikes in cpu utilization across hosts.", "analysis_config": { "bucket_span": "15m", "summary_count_field_name": "doc_count", diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json index 53667e3732c3a..186c9dcdb27e5 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json @@ -4,7 +4,7 @@ "hosts", "metrics" ], - "description": "", + "description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.", "analysis_config": { "bucket_span": "15m", "detectors": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json index 078221686f703..5470a44538e8e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json @@ -1,6 +1,6 @@ { "job_type": "anomaly_detector", - "description": "", + "description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.", "groups": [ "hosts", "metrics" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json index 5cd0c77a54125..601cc3807c441 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json @@ -1,6 +1,6 @@ { "job_type": "anomaly_detector", - "description": "", + "description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.", "groups": [ "hosts", "metrics" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json index 2b2e3adc779ef..92040217f91c5 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json @@ -1,7 +1,7 @@ { "id": "metrics_ui_k8s", "title": "Metrics Kubernetes", - "description": "", + "description": "Detect anomalous memory, cpu, and network behavior on kubernetes pods.", "type": "Metricbeat Data", "logoFile": "logo.json", "defaultIndexPattern": "metricbeat-*", diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json index 19f06920303a6..e854f27afdc8d 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json @@ -4,7 +4,7 @@ "k8s", "metrics" ], - "description": "", + "description": "Metrics: Kubernetes - Identify unusual spikes in cpu utilization across kubernetes pods.", "analysis_config": { "bucket_span": "15m", "detectors": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json index 15720a823e2a5..81593b3dd3930 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json @@ -4,7 +4,7 @@ "k8s", "metrics" ], - "description": "", + "description": "Metrics: Kubernetes - Identify unusual spikes in memory usage across kubernetes pods.", "analysis_config": { "bucket_span": "15m", "detectors": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json index 3b78270bc0ac4..d025cc9c2286a 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json @@ -1,6 +1,6 @@ { "job_type": "anomaly_detector", - "description": "", + "description": "Metrics: Kubernetes - Identify unusual spikes in inbound traffic across kubernetes pods.", "groups": [ "k8s", "metrics" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json index 4437f04c32c01..85fcb47d96119 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json @@ -1,6 +1,6 @@ { "job_type": "anomaly_detector", - "description": "", + "description": "Metrics: Kubernetes - Identify unusual spikes in outbound traffic across kubernetes pods.", "groups": [ "k8s", "metrics" From 2051546644e20ffd031947028b3ac9ee1bb305bb Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Tue, 15 Sep 2020 19:03:37 -0400 Subject: [PATCH 07/10] removes model plots --- .../modules/metrics_ui_hosts/ml/hosts_network_in.json | 3 --- .../modules/metrics_ui_k8s/ml/k8s_network_in.json | 3 --- .../modules/metrics_ui_k8s/ml/k8s_network_out.json | 3 --- 3 files changed, 9 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json index 5470a44538e8e..0054d90b1df33 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json @@ -19,9 +19,6 @@ ], "summary_count_field_name": "doc_count" }, - "model_plot_config": { - "enabled": "true" - }, "data_description": { "time_field": "@timestamp" }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json index d025cc9c2286a..b095bbd6b2a79 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json @@ -21,9 +21,6 @@ ], "summary_count_field_name": "doc_count" }, - "model_plot_config": { - "enabled": "true" - }, "data_description": { "time_field": "@timestamp" }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json index 85fcb47d96119..8fd712ab40b11 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json @@ -21,9 +21,6 @@ ], "summary_count_field_name": "doc_count" }, - "model_plot_config": { - "enabled": "true" - }, "data_description": { "time_field": "@timestamp" }, From 1248e2717e22912580e0e853f8ca9569790f947e Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Tue, 15 Sep 2020 19:08:05 -0400 Subject: [PATCH 08/10] updates terms agg sizes --- .../metrics_ui_hosts/ml/datafeed_hosts_network_in.json | 2 +- .../metrics_ui_hosts/ml/datafeed_hosts_network_out.json | 2 +- .../modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json | 4 ++-- .../modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json index 671d0ce185ad1..9c8218e0cbe0a 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json @@ -18,7 +18,7 @@ "time_span": "1000s" }, "aggregations": { - "host.name": {"terms": {"field": "host.name"}, + "host.name": {"terms": {"field": "host.name", "size": 1000}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json index e0435e99bbac1..e08158eef92a8 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json @@ -18,7 +18,7 @@ "time_span": "1000s" }, "aggregations": { - "host.name": {"terms": {"field": "host.name"}, + "host.name": {"terms": {"field": "host.name", "size": 1000}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json index 059184bc0ab12..a8a45a4179fcc 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json @@ -18,9 +18,9 @@ "time_span": "1000s" }, "aggregations": { - "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace"}, + "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 1000}, "aggregations": { - "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid"}, + "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 1000}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json index 35ae4201eadad..347963577c7ef 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json @@ -18,9 +18,9 @@ "time_span": "1000s" }, "aggregations": { - "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace"}, + "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 1000}, "aggregations": { - "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid"}, + "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 1000}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, From 1288b8fc52f187b9cfc690b1266c4f67680dba3e Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Tue, 15 Sep 2020 20:00:32 -0400 Subject: [PATCH 09/10] updates chunking config --- .../metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json | 8 ++++++-- .../metrics_ui_hosts/ml/datafeed_hosts_network_in.json | 4 ++-- .../metrics_ui_hosts/ml/datafeed_hosts_network_out.json | 4 ++-- .../metrics_ui_k8s/ml/datafeed_k8s_network_in.json | 6 +++--- .../metrics_ui_k8s/ml/datafeed_k8s_network_out.json | 6 +++--- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json index 3e0924041f4cb..66f48bad4d910 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json @@ -6,6 +6,10 @@ "indices_options": { "allow_no_indices": true }, + "chunking_config": { + "mode": "manual", + "time_span": "900s" + }, "query": { "bool": { "must": [ @@ -21,13 +25,13 @@ "host.name": { "terms": { "field": "host.name", - "size": 1000 + "size": 100 }, "aggs": { "buckets": { "date_histogram": { "field": "@timestamp", - "fixed_interval": "1m" + "fixed_interval": "5m" }, "aggs": { "@timestamp": { diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json index 9c8218e0cbe0a..7eb430632a81f 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json @@ -15,10 +15,10 @@ }, "chunking_config": { "mode": "manual", - "time_span": "1000s" + "time_span": "900s" }, "aggregations": { - "host.name": {"terms": {"field": "host.name", "size": 1000}, + "host.name": {"terms": {"field": "host.name", "size": 100}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json index e08158eef92a8..427cb678ce663 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json @@ -15,10 +15,10 @@ }, "chunking_config": { "mode": "manual", - "time_span": "1000s" + "time_span": "900s" }, "aggregations": { - "host.name": {"terms": {"field": "host.name", "size": 1000}, + "host.name": {"terms": {"field": "host.name", "size": 100}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json index a8a45a4179fcc..4fa4c603ea049 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json @@ -15,12 +15,12 @@ }, "chunking_config": { "mode": "manual", - "time_span": "1000s" + "time_span": "900s" }, "aggregations": { - "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 1000}, + "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25}, "aggregations": { - "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 1000}, + "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json index 347963577c7ef..633dd6bf490e7 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json @@ -15,12 +15,12 @@ }, "chunking_config": { "mode": "manual", - "time_span": "1000s" + "time_span": "900s" }, "aggregations": { - "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 1000}, + "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25}, "aggregations": { - "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 1000}, + "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100}, "aggregations": { "buckets": { "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, From 5f013319cc34f6862ad17f9e7c27e0f1385da257 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Wed, 16 Sep 2020 11:24:29 -0400 Subject: [PATCH 10/10] removes query and default index pattern from manifest, updates descriptions --- .../modules/metrics_ui_hosts/manifest.json | 23 +----- .../ml/datafeed_hosts_cpu_usage.json | 73 ------------------- .../metrics_ui_hosts/ml/hosts_cpu_usage.json | 51 ------------- .../modules/metrics_ui_k8s/manifest.json | 20 +---- .../ml/datafeed_k8s_cpu_usage.json | 17 ----- .../metrics_ui_k8s/ml/k8s_cpu_usage.json | 53 -------------- .../metrics_ui_k8s/ml/k8s_memory_usage.json | 4 +- .../metrics_ui_k8s/ml/k8s_network_in.json | 2 +- .../metrics_ui_k8s/ml/k8s_network_out.json | 2 +- 9 files changed, 6 insertions(+), 239 deletions(-) delete mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json delete mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json delete mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json delete mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json index e36ea2bdc2a17..29ac288c0649f 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json @@ -1,26 +1,10 @@ { "id": "metrics_ui_hosts", "title": "Metrics Hosts", - "description": "Detect anomalous memory, cpu, and network behavior on hosts.", + "description": "Detect anomalous memory and network behavior on hosts.", "type": "Metricbeat Data", "logoFile": "logo.json", - "defaultIndexPattern": "metricbeat-*", - "query": { - "bool": { - "must": [ - {"exists": {"field": "system"}} - ], - "must_not": [ - {"bool": {"filter": [{"exists":{"field": "kubernetes"}}]}}, - {"bool": {"filter": [{"exists":{"field": "docker"}}]}} - ] - } - }, "jobs": [ - { - "id": "hosts_cpu_usage", - "file": "hosts_cpu_usage.json" - }, { "id": "hosts_memory_usage", "file": "hosts_memory_usage.json" @@ -35,11 +19,6 @@ } ], "datafeeds": [ - { - "id": "datafeed-hosts_cpu_usage", - "file": "datafeed_hosts_cpu_usage.json", - "job_id": "hosts_cpu_usage" - }, { "id": "datafeed-hosts_memory_usage", "file": "datafeed_hosts_memory_usage.json", diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json deleted file mode 100644 index 66f48bad4d910..0000000000000 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "chunking_config": { - "mode": "manual", - "time_span": "900s" - }, - "query": { - "bool": { - "must": [ - { - "exists": { - "field": "system.cpu" - } - } - ] - } - }, - "aggs": { - "host.name": { - "terms": { - "field": "host.name", - "size": 100 - }, - "aggs": { - "buckets": { - "date_histogram": { - "field": "@timestamp", - "fixed_interval": "5m" - }, - "aggs": { - "@timestamp": { - "max": { - "field": "@timestamp" - } - }, - "cpu_user": { - "avg": { - "field": "system.cpu.user.pct" - } - }, - "cpu_system": { - "avg": { - "field": "system.cpu.system.pct" - } - }, - "cpu_cores": { - "max": { - "field": "system.cpu.cores" - } - }, - "cpu": { - "bucket_script": { - "buckets_path": { - "user": "cpu_user", - "system": "cpu_system", - "cores": "cpu_cores" - }, - "script": "(params.user + params.system) / params.cores", - "gap_policy": "skip" - } - } - } - } - } - } - } - } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json deleted file mode 100644 index 7a598f7502ea6..0000000000000 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "job_type": "anomaly_detector", - "groups": [ - "hosts", - "metrics" - ], - "description": "Metrics: Hosts - Identify unusual spikes in cpu utilization across hosts.", - "analysis_config": { - "bucket_span": "15m", - "summary_count_field_name": "doc_count", - "detectors": [ - { - "detector_description": "max('cpu')", - "function": "max", - "field_name": "cpu", - "custom_rules": [ - { - "actions": [ - "skip_result" - ], - "conditions": [ - { - "applies_to": "actual", - "operator": "lt", - "value": 0.1 - } - ] - } - ] - } - ], - "influencers": [ - "host.name" - ] - }, - "data_description": { - "time_field": "@timestamp" - }, - "analysis_limits": { - "model_memory_limit": "64mb" - }, - "custom_settings": { - "created_by": "ml-module-metrics-ui-hosts", - "custom_urls": [ - { - "url_name": "Host Metrics", - "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" - } - ] - } - } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json index 92040217f91c5..15336069e092b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json @@ -1,23 +1,10 @@ { "id": "metrics_ui_k8s", "title": "Metrics Kubernetes", - "description": "Detect anomalous memory, cpu, and network behavior on kubernetes pods.", + "description": "Detect anomalous memory and network behavior on Kubernetes pods.", "type": "Metricbeat Data", "logoFile": "logo.json", - "defaultIndexPattern": "metricbeat-*", - "query": { - "bool": { - "filter": [ - {"exists": {"field": "kubernetes"}}, - {"term": {"event.module": "kubernetes"}} - ] - } - }, "jobs": [ - { - "id": "k8s_cpu_usage", - "file": "k8s_cpu_usage.json" - }, { "id": "k8s_memory_usage", "file": "k8s_memory_usage.json" @@ -32,11 +19,6 @@ } ], "datafeeds": [ - { - "id": "datafeed-k8s_cpu_usage", - "file": "datafeed_k8s_cpu_usage.json", - "job_id": "k8s_cpu_usage" - }, { "id": "datafeed-k8s_memory_usage", "file": "datafeed_k8s_memory_usage.json", diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json deleted file mode 100644 index 8b9f7dafe2dda..0000000000000 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_cpu_usage.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "query": { - "bool": { - "must": [ - {"exists": {"field": "kubernetes.pod.uid"}}, - {"exists": {"field": "kubernetes.pod.cpu"}} - ] - } - } -} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json deleted file mode 100644 index e854f27afdc8d..0000000000000 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_cpu_usage.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "job_type": "anomaly_detector", - "groups": [ - "k8s", - "metrics" - ], - "description": "Metrics: Kubernetes - Identify unusual spikes in cpu utilization across kubernetes pods.", - "analysis_config": { - "bucket_span": "15m", - "detectors": [ - { - "detector_description": "max('kubernetes.pod.cpu.usage.node.pct') partitionfield='kubernetes.namespace'", - "function": "max", - "field_name": "kubernetes.pod.cpu.usage.node.pct", - "partition_field_name": "kubernetes.namespace", - "custom_rules": [ - { - "actions": [ - "skip_result" - ], - "conditions": [ - { - "applies_to": "actual", - "operator": "lt", - "value": 0.1 - } - ] - } - ] - } - ], - "influencers": [ - "kubernetes.namespace", - "kubernetes.node.name", - "kubernetes.pod.uid" - ] - }, - "data_description": { - "time_field": "@timestamp" - }, - "analysis_limits": { - "model_memory_limit": "64mb" - }, - "custom_settings": { - "created_by": "ml-module-metrics-ui-k8s", - "custom_urls": [ - { - "url_name": "Pod Metrics", - "url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" - } - ] - } - } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json index 81593b3dd3930..d3f58086e2fd5 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json @@ -4,12 +4,12 @@ "k8s", "metrics" ], - "description": "Metrics: Kubernetes - Identify unusual spikes in memory usage across kubernetes pods.", + "description": "Metrics: Kubernetes - Identify unusual spikes in memory usage across Kubernetes pods.", "analysis_config": { "bucket_span": "15m", "detectors": [ { - "detector_description": "max('kubernetes.pod.memory.usage.node.pct') partitionfield='kubernetes.namespace'", + "detector_description": "max('kubernetes.pod.memory.usage.node.pct')", "function": "max", "field_name": "kubernetes.pod.memory.usage.node.pct", "partition_field_name": "kubernetes.namespace", diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json index b095bbd6b2a79..212b2681beb77 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json @@ -1,6 +1,6 @@ { "job_type": "anomaly_detector", - "description": "Metrics: Kubernetes - Identify unusual spikes in inbound traffic across kubernetes pods.", + "description": "Metrics: Kubernetes - Identify unusual spikes in inbound traffic across Kubernetes pods.", "groups": [ "k8s", "metrics" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json index 8fd712ab40b11..b06b0ed5089ef 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json @@ -1,6 +1,6 @@ { "job_type": "anomaly_detector", - "description": "Metrics: Kubernetes - Identify unusual spikes in outbound traffic across kubernetes pods.", + "description": "Metrics: Kubernetes - Identify unusual spikes in outbound traffic across Kubernetes pods.", "groups": [ "k8s", "metrics"