From ea018b31c952b47bbc1060cc88890f537b563324 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 1 Oct 2020 09:08:24 +0100 Subject: [PATCH] [ML] Fixing module datafeed overrides (#78925) (#78999) --- x-pack/plugins/ml/common/types/modules.ts | 4 +- .../models/data_recognizer/data_recognizer.ts | 33 +++++- .../ml/log_entry_categories_count.json | 9 +- .../modules/metricbeat_system_ecs/logo.json | 2 +- .../metricbeat_system_ecs/manifest.json | 7 +- .../ml/datafeed_high_mean_cpu_iowait_ecs.json | 8 +- .../ml/datafeed_max_disk_utilization_ecs.json | 26 +++-- .../ml/datafeed_metricbeat_outages_ecs.json | 18 +-- .../ml/high_mean_cpu_iowait_ecs.json | 106 +++++++++--------- .../ml/max_disk_utilization_ecs.json | 106 +++++++++--------- .../ml/metricbeat_outages_ecs.json | 66 +++++------ .../modules/metrics_ui_hosts/logo.json | 4 +- .../modules/metrics_ui_hosts/manifest.json | 74 ++++++------ .../ml/datafeed_hosts_memory_usage.json | 30 ++--- .../ml/datafeed_hosts_network_in.json | 95 ++++++++++------ .../ml/datafeed_hosts_network_out.json | 95 ++++++++++------ .../ml/hosts_memory_usage.json | 96 ++++++++-------- .../metrics_ui_hosts/ml/hosts_network_in.json | 70 ++++++------ .../ml/hosts_network_out.json | 70 ++++++------ .../modules/metrics_ui_k8s/logo.json | 4 +- .../modules/metrics_ui_k8s/manifest.json | 74 ++++++------ .../ml/datafeed_k8s_memory_usage.json | 36 +++--- .../ml/datafeed_k8s_network_in.json | 103 +++++++++++------ .../ml/datafeed_k8s_network_out.json | 103 +++++++++++------ .../metrics_ui_k8s/ml/k8s_memory_usage.json | 102 ++++++++--------- .../metrics_ui_k8s/ml/k8s_network_in.json | 2 +- .../metrics_ui_k8s/ml/k8s_network_out.json | 2 +- 27 files changed, 756 insertions(+), 589 deletions(-) diff --git a/x-pack/plugins/ml/common/types/modules.ts b/x-pack/plugins/ml/common/types/modules.ts index bfa7e38332c1b..38f0c8cb9c117 100644 --- a/x-pack/plugins/ml/common/types/modules.ts +++ b/x-pack/plugins/ml/common/types/modules.ts @@ -11,7 +11,7 @@ export interface ModuleJob { config: Omit; } -export interface ModuleDataFeed { +export interface ModuleDatafeed { id: string; config: Omit; } @@ -49,7 +49,7 @@ export interface Module { defaultIndexPattern: string; query: any; jobs: ModuleJob[]; - datafeeds: ModuleDataFeed[]; + datafeeds: ModuleDatafeed[]; kibana: KibanaObjects; } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 141e78a91cf0d..96be6db03c52b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -17,7 +17,7 @@ import { MlInfoResponse } from '../../../common/types/ml_server_info'; import { KibanaObjects, KibanaObjectConfig, - ModuleDataFeed, + ModuleDatafeed, ModuleJob, Module, JobOverride, @@ -283,7 +283,7 @@ export class DataRecognizer { } const jobs: ModuleJob[] = []; - const datafeeds: ModuleDataFeed[] = []; + const datafeeds: ModuleDatafeed[] = []; const kibana: KibanaObjects = {}; // load all of the job configs await Promise.all( @@ -710,7 +710,7 @@ export class DataRecognizer { // save the datafeeds. // if any fail (e.g. it already exists), catch the error and mark the result // as success: false - async saveDatafeeds(datafeeds: ModuleDataFeed[]) { + async saveDatafeeds(datafeeds: ModuleDatafeed[]) { return await Promise.all( datafeeds.map(async (datafeed) => { try { @@ -723,7 +723,7 @@ export class DataRecognizer { ); } - async saveDatafeed(datafeed: ModuleDataFeed) { + async saveDatafeed(datafeed: ModuleDatafeed) { return this._asInternalUser.ml.putDatafeed( { datafeed_id: datafeed.id, @@ -734,7 +734,7 @@ export class DataRecognizer { } async startDatafeeds( - datafeeds: ModuleDataFeed[], + datafeeds: ModuleDatafeed[], start?: number, end?: number ): Promise<{ [key: string]: DatafeedResponse }> { @@ -746,7 +746,7 @@ export class DataRecognizer { } async startDatafeed( - datafeed: ModuleDataFeed, + datafeed: ModuleDatafeed, start: number | undefined, end: number | undefined ): Promise { @@ -1229,6 +1229,25 @@ export class DataRecognizer { const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides]; const { datafeeds } = moduleConfig; + // for some items in the datafeed, we should not merge. + // we should instead use the whole override object + function overwriteObjects(source: ModuleDatafeed['config'], update: DatafeedOverride) { + Object.entries(update).forEach(([key, val]) => { + if (typeof val === 'object') { + switch (key) { + case 'query': + case 'aggregations': + case 'aggs': + case 'script_fields': + source[key] = val as any; + break; + default: + break; + } + } + }); + } + // separate all the overrides. // the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module const generalOverrides: GeneralDatafeedsOverride[] = []; @@ -1244,6 +1263,7 @@ export class DataRecognizer { generalOverrides.forEach((o) => { datafeeds.forEach(({ config }) => { merge(config, o); + overwriteObjects(config, o); }); }); @@ -1259,6 +1279,7 @@ export class DataRecognizer { delete o.job_id; delete o.datafeed_id; merge(datafeed.config, o); + overwriteObjects(datafeed.config, o); } }); } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json index 40c47352371d4..ad7da3330bb6c 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json @@ -1,7 +1,9 @@ { "job_type": "anomaly_detector", "description": "Logs UI: Detects anomalies in count of log entries by category", - "groups": ["logs-ui"], + "groups": [ + "logs-ui" + ], "analysis_config": { "bucket_span": "15m", "categorization_field_name": "message", @@ -14,7 +16,10 @@ "use_null": true } ], - "influencers": ["event.dataset", "mlcategory"], + "influencers": [ + "event.dataset", + "mlcategory" + ], "per_partition_categorization": { "enabled": true, "stop_on_warn": false diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/logo.json index 04cfe48b16366..4b3e7343d6c0b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/logo.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/logo.json @@ -1,3 +1,3 @@ { - "icon": "metricbeatApp" + "icon": "metricbeatApp" } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/manifest.json index 499896ced7deb..6482fab429ecb 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/manifest.json @@ -8,7 +8,12 @@ "query": { "bool": { "filter": { - "terms" : { "event.dataset" : ["system.cpu", "system.filesystem"]} + "terms": { + "event.dataset": [ + "system.cpu", + "system.filesystem" + ] + } } } }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_high_mean_cpu_iowait_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_high_mean_cpu_iowait_ecs.json index fe87160142cff..b418f86296420 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_high_mean_cpu_iowait_ecs.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_high_mean_cpu_iowait_ecs.json @@ -6,10 +6,14 @@ "query": { "bool": { "filter": { - "term": { "event.dataset": "system.cpu" } + "term": { + "event.dataset": "system.cpu" + } }, "must": { - "exists": { "field": "system.cpu.iowait.pct" } + "exists": { + "field": "system.cpu.iowait.pct" + } } } } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_max_disk_utilization_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_max_disk_utilization_ecs.json index 6ccbfe94c220c..94a89f3dd0078 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_max_disk_utilization_ecs.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_max_disk_utilization_ecs.json @@ -1,16 +1,20 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "query": { - "bool": { - "filter": { - "term": { "event.dataset": "system.filesystem" } - }, - "must": { - "exists": { "field": "system.filesystem.used.pct" } + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": { + "term": { + "event.dataset": "system.filesystem" + } + }, + "must": { + "exists": { + "field": "system.filesystem.used.pct" } } } } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_metricbeat_outages_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_metricbeat_outages_ecs.json index d6f33127dfc08..d257068182826 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_metricbeat_outages_ecs.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_metricbeat_outages_ecs.json @@ -1,13 +1,15 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "query": { - "bool": { - "must": { - "exists": { "field": "event.dataset" } + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "must": { + "exists": { + "field": "event.dataset" } } } } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/high_mean_cpu_iowait_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/high_mean_cpu_iowait_ecs.json index b50338070d5cb..d6218acfae84f 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/high_mean_cpu_iowait_ecs.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/high_mean_cpu_iowait_ecs.json @@ -1,54 +1,56 @@ { - "job_type": "anomaly_detector", - "description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)", - "groups": ["metricbeat"], - "analysis_config": { - "bucket_span": "10m", - "detectors": [ - { - "detector_description": "high mean system.cpu.iowait.pct", - "function": "high_mean", - "field_name": "system.cpu.iowait.pct", - "partition_field_name": "host.name", - "custom_rules": [ - { - "actions": [ - "skip_result" - ], - "conditions": [ - { - "applies_to": "actual", - "operator": "lt", - "value": 0.25 - } - ] - } - ] - } - ], - "influencers": [ - "host.name" - ] - }, - "analysis_limits": { - "model_memory_limit": "25mb" - }, - "data_description": { - "time_field": "@timestamp", - "time_format": "epoch_ms" - }, - "custom_settings": { - "created_by": "ml-module-metricbeat-system", - "custom_urls": [ - { - "url_name": "Host overview", - "time_range": "3h", - "url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))" - }, - { - "url_name": "Raw data", - "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))" - } - ] - } + "job_type": "anomaly_detector", + "description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)", + "groups": [ + "metricbeat" + ], + "analysis_config": { + "bucket_span": "10m", + "detectors": [ + { + "detector_description": "high mean system.cpu.iowait.pct", + "function": "high_mean", + "field_name": "system.cpu.iowait.pct", + "partition_field_name": "host.name", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.25 + } + ] + } + ] + } + ], + "influencers": [ + "host.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "25mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-metricbeat-system", + "custom_urls": [ + { + "url_name": "Host overview", + "time_range": "3h", + "url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))" + } + ] } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/max_disk_utilization_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/max_disk_utilization_ecs.json index 65e515c38acd6..dd502c1d6ae37 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/max_disk_utilization_ecs.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/max_disk_utilization_ecs.json @@ -1,54 +1,56 @@ { - "job_type": "anomaly_detector", - "description": "Metricbeat filesystem: Detect unusual increases in disk utilization (ECS)", - "groups": ["metricbeat"], - "analysis_config": { - "bucket_span": "10m", - "detectors": [ - { - "detector_description": "max disk utilization", - "function": "max", - "field_name": "system.filesystem.used.pct", - "partition_field_name": "host.name", - "custom_rules": [ - { - "actions": [ - "skip_result" - ], - "conditions": [ - { - "applies_to": "actual", - "operator": "lt", - "value": 0.75 - } - ] - } - ] - } - ], - "influencers": [ - "host.name" - ] - }, - "analysis_limits": { - "model_memory_limit": "25mb" - }, - "data_description": { - "time_field": "@timestamp", - "time_format": "epoch_ms" - }, - "custom_settings": { - "created_by": "ml-module-metricbeat-system", - "custom_urls": [ - { - "url_name": "Host overview", - "time_range": "3h", - "url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))" - }, - { - "url_name": "Raw data", - "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.filesystem\u0022'),sort:!('@timestamp',desc))" - } - ] - } + "job_type": "anomaly_detector", + "description": "Metricbeat filesystem: Detect unusual increases in disk utilization (ECS)", + "groups": [ + "metricbeat" + ], + "analysis_config": { + "bucket_span": "10m", + "detectors": [ + { + "detector_description": "max disk utilization", + "function": "max", + "field_name": "system.filesystem.used.pct", + "partition_field_name": "host.name", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.75 + } + ] + } + ] + } + ], + "influencers": [ + "host.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "25mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-metricbeat-system", + "custom_urls": [ + { + "url_name": "Host overview", + "time_range": "3h", + "url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.filesystem\u0022'),sort:!('@timestamp',desc))" + } + ] } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/metricbeat_outages_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/metricbeat_outages_ecs.json index f93419c68c67d..fb2838ac57856 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/metricbeat_outages_ecs.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/metricbeat_outages_ecs.json @@ -1,34 +1,36 @@ { - "job_type": "anomaly_detector", - "description": "Metricbeat outages: Detect unusual decreases in metricbeat documents (ECS)", - "groups": ["metricbeat"], - "analysis_config": { - "bucket_span": "10m", - "detectors": [ - { - "detector_description": "low_count", - "function": "low_count", - "partition_field_name": "event.dataset" - } - ], - "influencers": [ - "event.dataset" - ] - }, - "analysis_limits": { - "model_memory_limit": "15mb" - }, - "data_description": { - "time_field": "@timestamp", - "time_format": "epoch_ms" - }, - "custom_settings": { - "created_by": "ml-module-metricbeat-system", - "custom_urls": [ - { - "url_name": "Raw data", - "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:''),sort:!('@timestamp',desc))" - } - ] - } + "job_type": "anomaly_detector", + "description": "Metricbeat outages: Detect unusual decreases in metricbeat documents (ECS)", + "groups": [ + "metricbeat" + ], + "analysis_config": { + "bucket_span": "10m", + "detectors": [ + { + "detector_description": "low_count", + "function": "low_count", + "partition_field_name": "event.dataset" + } + ], + "influencers": [ + "event.dataset" + ] + }, + "analysis_limits": { + "model_memory_limit": "15mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-metricbeat-system", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:''),sort:!('@timestamp',desc))" + } + ] } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json index 2e57038bbc639..206f34df4f50f 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json @@ -1,3 +1,3 @@ { - "icon": "metricsApp" -} \ No newline at end of file + "icon": "metricsApp" +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json index 29ac288c0649f..dd1e191fda47b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json @@ -1,38 +1,38 @@ { - "id": "metrics_ui_hosts", - "title": "Metrics Hosts", - "description": "Detect anomalous memory and network behavior on hosts.", - "type": "Metricbeat Data", - "logoFile": "logo.json", - "jobs": [ - { - "id": "hosts_memory_usage", - "file": "hosts_memory_usage.json" - }, - { - "id": "hosts_network_in", - "file": "hosts_network_in.json" - }, - { - "id": "hosts_network_out", - "file": "hosts_network_out.json" - } - ], - "datafeeds": [ - { - "id": "datafeed-hosts_memory_usage", - "file": "datafeed_hosts_memory_usage.json", - "job_id": "hosts_memory_usage" - }, - { - "id": "datafeed-hosts_network_in", - "file": "datafeed_hosts_network_in.json", - "job_id": "hosts_network_in" - }, - { - "id": "datafeed-hosts_network_out", - "file": "datafeed_hosts_network_out.json", - "job_id": "hosts_network_out" - } - ] -} \ No newline at end of file + "id": "metrics_ui_hosts", + "title": "Metrics Hosts", + "description": "Detect anomalous memory and network behavior on hosts.", + "type": "Metricbeat Data", + "logoFile": "logo.json", + "jobs": [ + { + "id": "hosts_memory_usage", + "file": "hosts_memory_usage.json" + }, + { + "id": "hosts_network_in", + "file": "hosts_network_in.json" + }, + { + "id": "hosts_network_out", + "file": "hosts_network_out.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-hosts_memory_usage", + "file": "datafeed_hosts_memory_usage.json", + "job_id": "hosts_memory_usage" + }, + { + "id": "datafeed-hosts_network_in", + "file": "datafeed_hosts_network_in.json", + "job_id": "hosts_network_in" + }, + { + "id": "datafeed-hosts_network_out", + "file": "datafeed_hosts_network_out.json", + "job_id": "hosts_network_out" + } + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json index db883a6ce36f9..0fdede28563a3 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json @@ -1,16 +1,20 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "query": { - "bool": { - "must": [ - {"exists": {"field": "system.memory"}} - ] + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "indices_options": { + "allow_no_indices": true + }, + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "system.memory" + } } + ] } -} \ No newline at end of file + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json index 7eb430632a81f..d4cfbebe0a437 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json @@ -1,40 +1,65 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "query": { - "bool": { - "must": [ - {"exists": {"field": "system.network"}} - ] + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "indices_options": { + "allow_no_indices": true + }, + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "system.network" + } } - }, - "chunking_config": { - "mode": "manual", - "time_span": "900s" - }, - "aggregations": { - "host.name": {"terms": {"field": "host.name", "size": 100}, - "aggregations": { - "buckets": { - "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, - "aggregations": { - "@timestamp": {"max": {"field": "@timestamp"}}, - "bytes_in_max": {"max": {"field": "system.network.in.bytes"}}, - "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}}, - "positive_only":{ - "bucket_script": { - "buckets_path": {"in_derivative": "bytes_in_derivative.value"}, - "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" - } - } - } - } + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "900s" + }, + "aggregations": { + "host.name": { + "terms": { + "field": "host.name", + "size": 100 + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "5m" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "bytes_in_max": { + "max": { + "field": "system.network.in.bytes" + } + }, + "bytes_in_derivative": { + "derivative": { + "buckets_path": "bytes_in_max" + } + }, + "positive_only": { + "bucket_script": { + "buckets_path": { + "in_derivative": "bytes_in_derivative.value" + }, + "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" + } } + } } + } } -} \ No newline at end of file + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json index 427cb678ce663..903cf31416ef4 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json @@ -1,40 +1,65 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "query": { - "bool": { - "must": [ - {"exists": {"field": "system.network"}} - ] + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "indices_options": { + "allow_no_indices": true + }, + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "system.network" + } } - }, - "chunking_config": { - "mode": "manual", - "time_span": "900s" - }, - "aggregations": { - "host.name": {"terms": {"field": "host.name", "size": 100}, - "aggregations": { - "buckets": { - "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, - "aggregations": { - "@timestamp": {"max": {"field": "@timestamp"}}, - "bytes_out_max": {"max": {"field": "system.network.out.bytes"}}, - "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}}, - "positive_only":{ - "bucket_script": { - "buckets_path": {"out_derivative": "bytes_out_derivative.value"}, - "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0" - } - } - } - } + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "900s" + }, + "aggregations": { + "host.name": { + "terms": { + "field": "host.name", + "size": 100 + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "5m" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "bytes_out_max": { + "max": { + "field": "system.network.out.bytes" + } + }, + "bytes_out_derivative": { + "derivative": { + "buckets_path": "bytes_out_max" + } + }, + "positive_only": { + "bucket_script": { + "buckets_path": { + "out_derivative": "bytes_out_derivative.value" + }, + "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0" + } } + } } + } } -} \ No newline at end of file + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json index 186c9dcdb27e5..c5f62105613ba 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json @@ -1,50 +1,50 @@ { - "job_type": "anomaly_detector", - "groups": [ - "hosts", - "metrics" - ], - "description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.", - "analysis_config": { - "bucket_span": "15m", - "detectors": [ - { - "detector_description": "max('system.memory.actual.used.pct')", - "function": "max", - "field_name": "system.memory.actual.used.pct", - "custom_rules": [ - { - "actions": [ - "skip_result" - ], - "conditions": [ - { - "applies_to": "actual", - "operator": "lt", - "value": 0.1 - } - ] - } + "job_type": "anomaly_detector", + "groups": [ + "hosts", + "metrics" + ], + "description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max('system.memory.actual.used.pct')", + "function": "max", + "field_name": "system.memory.actual.used.pct", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.1 + } ] - } - ], - "influencers": [ - "host.name" - ] - }, - "data_description": { - "time_field": "@timestamp" - }, - "analysis_limits": { - "model_memory_limit": "64mb" - }, - "custom_settings": { - "created_by": "ml-module-metrics-ui-hosts", - "custom_urls": [ - { - "url_name": "Host Metrics", - "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" - } - ] - } - } \ No newline at end of file + } + ] + } + ], + "influencers": [ + "host.name" + ] + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-ui-hosts", + "custom_urls": [ + { + "url_name": "Host Metrics", + "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json index 0054d90b1df33..258fb87f5260c 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json @@ -1,37 +1,37 @@ { - "job_type": "anomaly_detector", - "description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.", - "groups": [ - "hosts", - "metrics" + "job_type": "anomaly_detector", + "description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.", + "groups": [ + "hosts", + "metrics" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max(bytes_in_derivative)", + "function": "max", + "field_name": "bytes_in_derivative" + } ], - "analysis_config": { - "bucket_span": "15m", - "detectors": [ - { - "detector_description": "max(bytes_in_derivative)", - "function": "max", - "field_name": "bytes_in_derivative" - } - ], - "influencers": [ - "host.name" - ], - "summary_count_field_name": "doc_count" - }, - "data_description": { - "time_field": "@timestamp" - }, - "analysis_limits": { - "model_memory_limit": "32mb" - }, - "custom_settings": { - "created_by": "ml-module-metrics-ui-hosts", - "custom_urls": [ - { - "url_name": "Host Metrics", - "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" - } - ] - } - } \ No newline at end of file + "influencers": [ + "host.name" + ], + "summary_count_field_name": "doc_count" + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-ui-hosts", + "custom_urls": [ + { + "url_name": "Host Metrics", + "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json index 601cc3807c441..381bc09bac46c 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json @@ -1,37 +1,37 @@ { - "job_type": "anomaly_detector", - "description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.", - "groups": [ - "hosts", - "metrics" + "job_type": "anomaly_detector", + "description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.", + "groups": [ + "hosts", + "metrics" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max(bytes_out_derivative)", + "function": "max", + "field_name": "bytes_out_derivative" + } ], - "analysis_config": { - "bucket_span": "15m", - "detectors": [ - { - "detector_description": "max(bytes_out_derivative)", - "function": "max", - "field_name": "bytes_out_derivative" - } - ], - "influencers": [ - "host.name" - ], - "summary_count_field_name": "doc_count" - }, - "data_description": { - "time_field": "@timestamp" - }, - "analysis_limits": { - "model_memory_limit": "32mb" - }, - "custom_settings": { - "created_by": "ml-module-metrics-ui-hosts", - "custom_urls": [ - { - "url_name": "Host Metrics", - "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" - } - ] - } - } \ No newline at end of file + "influencers": [ + "host.name" + ], + "summary_count_field_name": "doc_count" + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-ui-hosts", + "custom_urls": [ + { + "url_name": "Host Metrics", + "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json index 63105a28c0ab1..206f34df4f50f 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json @@ -1,3 +1,3 @@ { - "icon": "metricsApp" -} \ No newline at end of file + "icon": "metricsApp" +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json index 15336069e092b..e3a02120a4ac5 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json @@ -1,38 +1,38 @@ { - "id": "metrics_ui_k8s", - "title": "Metrics Kubernetes", - "description": "Detect anomalous memory and network behavior on Kubernetes pods.", - "type": "Metricbeat Data", - "logoFile": "logo.json", - "jobs": [ - { - "id": "k8s_memory_usage", - "file": "k8s_memory_usage.json" - }, - { - "id": "k8s_network_in", - "file": "k8s_network_in.json" - }, - { - "id": "k8s_network_out", - "file": "k8s_network_out.json" - } - ], - "datafeeds": [ - { - "id": "datafeed-k8s_memory_usage", - "file": "datafeed_k8s_memory_usage.json", - "job_id": "k8s_memory_usage" - }, - { - "id": "datafeed-k8s_network_in", - "file": "datafeed_k8s_network_in.json", - "job_id": "k8s_network_in" - }, - { - "id": "datafeed-k8s_network_out", - "file": "datafeed_k8s_network_out.json", - "job_id": "k8s_network_out" - } - ] - } \ No newline at end of file + "id": "metrics_ui_k8s", + "title": "Metrics Kubernetes", + "description": "Detect anomalous memory and network behavior on Kubernetes pods.", + "type": "Metricbeat Data", + "logoFile": "logo.json", + "jobs": [ + { + "id": "k8s_memory_usage", + "file": "k8s_memory_usage.json" + }, + { + "id": "k8s_network_in", + "file": "k8s_network_in.json" + }, + { + "id": "k8s_network_out", + "file": "k8s_network_out.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-k8s_memory_usage", + "file": "datafeed_k8s_memory_usage.json", + "job_id": "k8s_memory_usage" + }, + { + "id": "datafeed-k8s_network_in", + "file": "datafeed_k8s_network_in.json", + "job_id": "k8s_network_in" + }, + { + "id": "datafeed-k8s_network_out", + "file": "datafeed_k8s_network_out.json", + "job_id": "k8s_network_out" + } + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json index 14590f743528e..1a443dc6f3bb7 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json @@ -1,17 +1,25 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "query": { - "bool": { - "must": [ - {"exists": {"field": "kubernetes.pod.uid"}}, - {"exists": {"field": "kubernetes.pod.memory"}} - ] + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "indices_options": { + "allow_no_indices": true + }, + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "kubernetes.pod.uid" + } + }, + { + "exists": { + "field": "kubernetes.pod.memory" + } } + ] } -} \ No newline at end of file + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json index 4fa4c603ea049..18a48b300f202 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json @@ -1,44 +1,73 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "query": { - "bool": { - "must": [ - {"exists": {"field": "kubernetes.pod.network"}} - ] + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "indices_options": { + "allow_no_indices": true + }, + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "kubernetes.pod.network" + } } - }, - "chunking_config": { - "mode": "manual", - "time_span": "900s" - }, - "aggregations": { - "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25}, - "aggregations": { - "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100}, - "aggregations": { - "buckets": { - "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, - "aggregations": { - "@timestamp": {"max": {"field": "@timestamp"}}, - "bytes_in_max": {"max": {"field": "kubernetes.pod.network.rx.bytes"}}, - "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}}, - "positive_only":{ - "bucket_script": { - "buckets_path": {"in_derivative": "bytes_in_derivative.value"}, - "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" - } - } - } - } - } + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "900s" + }, + "aggregations": { + "kubernetes.namespace": { + "terms": { + "field": "kubernetes.namespace", + "size": 25 + }, + "aggregations": { + "kubernetes.pod.uid": { + "terms": { + "field": "kubernetes.pod.uid", + "size": 100 + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "5m" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "bytes_in_max": { + "max": { + "field": "kubernetes.pod.network.rx.bytes" + } + }, + "bytes_in_derivative": { + "derivative": { + "buckets_path": "bytes_in_max" + } + }, + "positive_only": { + "bucket_script": { + "buckets_path": { + "in_derivative": "bytes_in_derivative.value" + }, + "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" + } } + } } + } } + } } + } } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json index 633dd6bf490e7..18a3f93195095 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json @@ -1,44 +1,73 @@ { - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "indices_options": { - "allow_no_indices": true - }, - "query": { - "bool": { - "must": [ - {"exists": {"field": "kubernetes.pod.network"}} - ] + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "indices_options": { + "allow_no_indices": true + }, + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "kubernetes.pod.network" + } } - }, - "chunking_config": { - "mode": "manual", - "time_span": "900s" - }, - "aggregations": { - "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25}, - "aggregations": { - "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100}, - "aggregations": { - "buckets": { - "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, - "aggregations": { - "@timestamp": {"max": {"field": "@timestamp"}}, - "bytes_out_max": {"max": {"field": "kubernetes.pod.network.tx.bytes"}}, - "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}}, - "positive_only":{ - "bucket_script": { - "buckets_path": {"pos_derivative": "bytes_out_derivative.value"}, - "script": "params.pos_derivative > 0.0 ? params.pos_derivative : 0.0" - } - } - } - } - } + ] + } + }, + "chunking_config": { + "mode": "manual", + "time_span": "900s" + }, + "aggregations": { + "kubernetes.namespace": { + "terms": { + "field": "kubernetes.namespace", + "size": 25 + }, + "aggregations": { + "kubernetes.pod.uid": { + "terms": { + "field": "kubernetes.pod.uid", + "size": 100 + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "5m" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "bytes_out_max": { + "max": { + "field": "kubernetes.pod.network.tx.bytes" + } + }, + "bytes_out_derivative": { + "derivative": { + "buckets_path": "bytes_out_max" + } + }, + "positive_only": { + "bucket_script": { + "buckets_path": { + "pos_derivative": "bytes_out_derivative.value" + }, + "script": "params.pos_derivative > 0.0 ? params.pos_derivative : 0.0" + } } + } } + } } + } } + } } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json index d3f58086e2fd5..ef57612e9f90e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json @@ -1,53 +1,53 @@ { - "job_type": "anomaly_detector", - "groups": [ - "k8s", - "metrics" + "job_type": "anomaly_detector", + "groups": [ + "k8s", + "metrics" + ], + "description": "Metrics: Kubernetes - Identify unusual spikes in memory usage across Kubernetes pods.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "max('kubernetes.pod.memory.usage.node.pct')", + "function": "max", + "field_name": "kubernetes.pod.memory.usage.node.pct", + "partition_field_name": "kubernetes.namespace", + "custom_rules": [ + { + "actions": [ + "skip_result" + ], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.1 + } + ] + } + ] + } ], - "description": "Metrics: Kubernetes - Identify unusual spikes in memory usage across Kubernetes pods.", - "analysis_config": { - "bucket_span": "15m", - "detectors": [ - { - "detector_description": "max('kubernetes.pod.memory.usage.node.pct')", - "function": "max", - "field_name": "kubernetes.pod.memory.usage.node.pct", - "partition_field_name": "kubernetes.namespace", - "custom_rules": [ - { - "actions": [ - "skip_result" - ], - "conditions": [ - { - "applies_to": "actual", - "operator": "lt", - "value": 0.1 - } - ] - } - ] - } - ], - "influencers": [ - "kubernetes.namespace", - "kubernetes.node.name", - "kubernetes.pod.uid" - ] - }, - "data_description": { - "time_field": "@timestamp" - }, - "analysis_limits": { - "model_memory_limit": "64mb" - }, - "custom_settings": { - "created_by": "ml-module-metrics-ui-k8s", - "custom_urls": [ - { - "url_name": "Pod Metrics", - "url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" - } - ] - } - } \ No newline at end of file + "influencers": [ + "kubernetes.namespace", + "kubernetes.node.name", + "kubernetes.pod.uid" + ] + }, + "data_description": { + "time_field": "@timestamp" + }, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "custom_settings": { + "created_by": "ml-module-metrics-ui-k8s", + "custom_urls": [ + { + "url_name": "Pod Metrics", + "url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json index 212b2681beb77..91f855a59add5 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json @@ -18,7 +18,7 @@ "influencers": [ "kubernetes.namespace", "kubernetes.pod.uid" - ], + ], "summary_count_field_name": "doc_count" }, "data_description": { diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json index b06b0ed5089ef..e68866a655acf 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json @@ -18,7 +18,7 @@ "influencers": [ "kubernetes.namespace", "kubernetes.pod.uid" - ], + ], "summary_count_field_name": "doc_count" }, "data_description": {