[ML] Adds ML modules for Metrics UI Integration (#76460)

* adds metrics ml integration * renames jobs, updates datafeeds * adds allow_no_indices: true for datafeeds * updates module ids in manifest * adds custom urls * adds module and individual job descriptions * removes model plots * updates terms agg sizes * updates chunking config * removes query and default index pattern from manifest, updates descriptions Co-authored-by: Elastic Machine <[email protected]>
elastic · Sep 17, 2020 · 6d12c68 · 6d12c68
1 parent 0cde5fd
commit 6d12c68
Show file tree

Hide file tree

Showing 17 changed files with 540 additions and 0 deletions.
diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json
@@ -0,0 +1,3 @@
+{
+    "icon": "metricsApp"
+}
diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json
@@ -0,0 +1,38 @@
+{
+    "id": "metrics_ui_hosts",
+    "title": "Metrics Hosts",
+    "description": "Detect anomalous memory and network behavior on hosts.",
+    "type": "Metricbeat Data",
+    "logoFile": "logo.json",
+    "jobs": [
+        {
+            "id": "hosts_memory_usage",
+            "file": "hosts_memory_usage.json"
+        },
+        {
+            "id": "hosts_network_in",
+            "file": "hosts_network_in.json"
+        },
+        {
+            "id": "hosts_network_out",
+            "file": "hosts_network_out.json"
+        }
+    ],
+    "datafeeds": [
+        {
+            "id": "datafeed-hosts_memory_usage",
+            "file": "datafeed_hosts_memory_usage.json",
+            "job_id": "hosts_memory_usage"
+        },
+        {
+            "id": "datafeed-hosts_network_in",
+            "file": "datafeed_hosts_network_in.json",
+            "job_id": "hosts_network_in"
+        },
+        {
+            "id": "datafeed-hosts_network_out",
+            "file": "datafeed_hosts_network_out.json",
+            "job_id": "hosts_network_out"
+        }
+    ]
+}
diff --git a/...erver/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json b/...erver/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json
@@ -0,0 +1,16 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "system.memory"}}
+            ]
+        }
+    }
+}
diff --git a/.../server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json b/.../server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json
@@ -0,0 +1,40 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "system.network"}}
+            ]
+        }
+    },
+    "chunking_config": {
+        "mode": "manual",
+        "time_span": "900s"
+    },
+    "aggregations": {
+        "host.name": {"terms": {"field": "host.name", "size": 100},
+            "aggregations": {
+                "buckets": {
+                    "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+                    "aggregations": {
+                        "@timestamp": {"max": {"field": "@timestamp"}},
+                        "bytes_in_max": {"max": {"field": "system.network.in.bytes"}},
+                        "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
+                        "positive_only":{
+                            "bucket_script": {
+                                "buckets_path": {"in_derivative": "bytes_in_derivative.value"},
+                                "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/...server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json b/...server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json
@@ -0,0 +1,40 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "system.network"}}
+            ]
+        }
+    },
+    "chunking_config": {
+        "mode": "manual",
+        "time_span": "900s"
+    },
+    "aggregations": {
+        "host.name": {"terms": {"field": "host.name", "size": 100},
+            "aggregations": {
+                "buckets": {
+                    "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+                    "aggregations": {
+                        "@timestamp": {"max": {"field": "@timestamp"}},
+                        "bytes_out_max": {"max": {"field": "system.network.out.bytes"}},
+                        "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}},
+                        "positive_only":{
+                            "bucket_script": {
+                                "buckets_path": {"out_derivative": "bytes_out_derivative.value"},
+                                "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/...gins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json b/...gins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json
@@ -0,0 +1,50 @@
+{
+    "job_type": "anomaly_detector",
+    "groups": [
+      "hosts",
+      "metrics"
+    ],
+    "description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.",
+    "analysis_config": {
+      "bucket_span": "15m",
+      "detectors": [
+        {
+            "detector_description": "max('system.memory.actual.used.pct')",
+            "function": "max",
+            "field_name": "system.memory.actual.used.pct",
+            "custom_rules": [
+                {
+                    "actions": [
+                    "skip_result"
+                    ],
+                    "conditions": [
+                        {
+                            "applies_to": "actual",
+                            "operator": "lt",
+                            "value": 0.1
+                        }
+                    ]
+                }
+            ]
+        }
+      ],
+      "influencers": [
+        "host.name"
+      ]
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "analysis_limits": {
+      "model_memory_limit": "64mb"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-metrics-ui-hosts",
+      "custom_urls": [
+        {
+          "url_name": "Host Metrics",
+          "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+        }
+      ]
+    }
+  }
diff --git a/...lugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json b/...lugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json
@@ -0,0 +1,37 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.",
+    "groups": [
+      "hosts",
+      "metrics"
+    ],
+    "analysis_config": {
+      "bucket_span": "15m",
+      "detectors": [
+        {
+          "detector_description": "max(bytes_in_derivative)",
+          "function": "max",
+          "field_name": "bytes_in_derivative"
+        }
+      ],
+      "influencers": [
+        "host.name"
+        ],
+      "summary_count_field_name": "doc_count"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "analysis_limits": {
+      "model_memory_limit": "32mb"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-metrics-ui-hosts",
+      "custom_urls": [
+        {
+          "url_name": "Host Metrics",
+          "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+        }
+      ]
+    }
+  }
diff --git a/...ugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json b/...ugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json
@@ -0,0 +1,37 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.",
+    "groups": [
+      "hosts",
+      "metrics"
+    ],
+    "analysis_config": {
+      "bucket_span": "15m",
+      "detectors": [
+        {
+          "detector_description": "max(bytes_out_derivative)",
+          "function": "max",
+          "field_name": "bytes_out_derivative"
+        }
+      ],
+      "influencers": [
+        "host.name"
+        ],
+      "summary_count_field_name": "doc_count"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "analysis_limits": {
+      "model_memory_limit": "32mb"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-metrics-ui-hosts",
+      "custom_urls": [
+        {
+          "url_name": "Host Metrics",
+          "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+        }
+      ]
+    }
+  }
diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json
@@ -0,0 +1,3 @@
+{
+	"icon": "metricsApp"
+}
diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json
@@ -0,0 +1,38 @@
+{
+    "id": "metrics_ui_k8s",
+    "title": "Metrics Kubernetes",
+    "description": "Detect anomalous memory and network behavior on Kubernetes pods.",
+    "type": "Metricbeat Data",
+    "logoFile": "logo.json",
+    "jobs": [
+      {
+        "id": "k8s_memory_usage",
+        "file": "k8s_memory_usage.json"
+      },
+      {
+        "id": "k8s_network_in",
+        "file": "k8s_network_in.json"
+      },
+      {
+        "id": "k8s_network_out",
+        "file": "k8s_network_out.json"
+      }
+    ],
+    "datafeeds": [
+      {
+        "id": "datafeed-k8s_memory_usage",
+        "file": "datafeed_k8s_memory_usage.json",
+        "job_id": "k8s_memory_usage"
+      },
+      {
+        "id": "datafeed-k8s_network_in",
+        "file": "datafeed_k8s_network_in.json",
+        "job_id": "k8s_network_in"
+      },
+      {
+        "id": "datafeed-k8s_network_out",
+        "file": "datafeed_k8s_network_out.json",
+        "job_id": "k8s_network_out"
+      }
+    ]
+  }
diff --git a/...ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json b/...ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json
@@ -0,0 +1,17 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "kubernetes.pod.uid"}},
+                {"exists": {"field": "kubernetes.pod.memory"}}
+            ]
+        }
+    }
+}
diff --git a/...s/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json b/...s/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json
@@ -0,0 +1,44 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "kubernetes.pod.network"}}
+            ]
+        }
+    },
+    "chunking_config": {
+        "mode": "manual",
+        "time_span": "900s"
+    },
+    "aggregations": {
+        "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25},
+            "aggregations": {
+                "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100},
+                    "aggregations": {
+                        "buckets": {
+                            "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+                            "aggregations": {
+                                "@timestamp": {"max": {"field": "@timestamp"}},
+                                "bytes_in_max": {"max": {"field": "kubernetes.pod.network.rx.bytes"}},
+                                "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
+                                "positive_only":{
+                                    "bucket_script": {
+                                        "buckets_path": {"in_derivative": "bytes_in_derivative.value"},
+                                        "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}