diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index e6007695564..c619b5f4dd8 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -71,7 +71,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d - Add experimental Redis module. {pull}4441[4441] - Nginx module: use the first not-private IP address as the remote_ip. {pull}4417[4417] - Load Ingest Node pipelines when the Elasticsearch connection is established, instead of only once at startup. {pull}4479[4479] -- Add support for loading Xpack Machine Learning configurations from the modules, and added sample configurations for the Nginx module. {pull}4506[4506] +- Add support for loading Xpack Machine Learning configurations from the modules, and added sample configurations for the Nginx module. {pull}4506[4506] {pull}4609[4609] + - Add udp prospector type. {pull}4452[4452] - Enabled Cgo which means libc is dynamically compiled. {pull}4546[4546] - Add Beta module config reloading mechanism {pull}4566[4566] diff --git a/filebeat/docs/images/filebeat-nginx-ml.png b/filebeat/docs/images/filebeat-nginx-ml.png new file mode 100644 index 00000000000..ff3505c30e8 Binary files /dev/null and b/filebeat/docs/images/filebeat-nginx-ml.png differ diff --git a/filebeat/module/nginx/_meta/kibana/5.x/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer.json b/filebeat/module/nginx/_meta/kibana/5.x/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer.json new file mode 100644 index 00000000000..28683287eb5 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "ML Nginx Access Remote IP Count Explorer", + "uiStateJSON": "{\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Remote-IP-Timechart\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Response-Code-Timechart\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-Remote-IPs-Table\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Map\",\"col\":7,\"row\":4},{\"size_x\":12,\"size_y\":9,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/dashboard/ML-Nginx-Remote-IP-URL-Explorer.json b/filebeat/module/nginx/_meta/kibana/5.x/dashboard/ML-Nginx-Remote-IP-URL-Explorer.json new file mode 100644 index 00000000000..24a4a33fd91 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/dashboard/ML-Nginx-Remote-IP-URL-Explorer.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "ML Nginx Access Remote IP URL Explorer", + "uiStateJSON": "{\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"col\":1,\"id\":\"ML-Nginx-Access-Unique-Count-URL-Timechart\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Nginx-Access-Response-Code-Timechart\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ML-Nginx-Access-Top-Remote-IPs-Table\",\"panelIndex\":3,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Nginx-Access-Map\",\"panelIndex\":4,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":8,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/search/ML-Filebeat-Nginx-Access.json b/filebeat/module/nginx/_meta/kibana/5.x/search/ML-Filebeat-Nginx-Access.json new file mode 100644 index 00000000000..b97183fee41 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/search/ML-Filebeat-Nginx-Access.json @@ -0,0 +1,16 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "Filebeat Nginx Access Data", + "title": "ML Nginx Access Data", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:nginx.access\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "_source" + ] +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Map.json b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Map.json new file mode 100644 index 00000000000..f341e7f6177 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Map.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"nginx.access.geoip.location\"},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"listeners\":{},\"params\":{\"addTooltip\":true,\"heatBlur\":15,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatNormalizeData\":true,\"heatRadius\":25,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[15,5],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"enabled\":false,\"options\":{\"attribution\":\"Maps provided by USGS\",\"format\":\"image/png\",\"layers\":\"0\",\"styles\":\"\",\"transparent\":true,\"version\":\"1.3.0\"},\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"}},\"title\":\"ML Nginx Access Map\",\"type\":\"tile_map\"}", + "description": "", + "title": "ML Nginx Access Map", + "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", + "version": 1, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Remote-IP-Timechart.json b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Remote-IP-Timechart.json new file mode 100644 index 00000000000..0635b3ddd5d --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Remote-IP-Timechart.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML Nginx Access Remote IP Timechart\",\"type\":\"area\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 5 minutes\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"area\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"nginx.access.remote_ip\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML Nginx Access Remote IP Timechart", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "version": 1, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Response-Code-Timechart.json b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Response-Code-Timechart.json new file mode 100644 index 00000000000..7266db3ea83 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Response-Code-Timechart.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML Nginx Access Response Code Timechart\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"nginx.access.response_code\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML Nginx Access Response Code Timechart", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", + "version": 1, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Top-Remote-IPs-Table.json b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Top-Remote-IPs-Table.json new file mode 100644 index 00000000000..7d6166664f1 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Top-Remote-IPs-Table.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML Nginx Access Top Remote IPs Table\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.remote_ip\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML Nginx Access Top Remote IPs Table", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Top-URLs-Table.json b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Top-URLs-Table.json new file mode 100644 index 00000000000..e5336a19df7 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Top-URLs-Table.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML Nginx Access Top URLs Table\",\"type\":\"table\",\"params\":{\"perPage\":100,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.url\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML Nginx Access Top URLs Table", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Unique-Count-URL-Timechart.json b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Unique-Count-URL-Timechart.json new file mode 100644 index 00000000000..d663f45a5ff --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/5.x/visualization/ML-Nginx-Access-Unique-Count-URL-Timechart.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML Nginx Access Unique Count URL Timechart\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per day\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of nginx.access.url\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"line\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Unique count of nginx.access.url\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"nginx.access.url\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "description": "", + "title": "ML Nginx Access Unique Count URL Timechart", + "uiStateJSON": "{}", + "version": 1, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/default/dashboard/ml-nginx-access-remote-ip-count-explorer.json b/filebeat/module/nginx/_meta/kibana/default/dashboard/ml-nginx-access-remote-ip-count-explorer.json new file mode 100644 index 00000000000..3f08810a3d4 --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/default/dashboard/ml-nginx-access-remote-ip-count-explorer.json @@ -0,0 +1,124 @@ +{ + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Remote IP Timechart", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Remote IP Timechart\",\"type\":\"area\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 5 minutes\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"area\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"nginx.access.remote_ip\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Remote-IP-Timechart", + "type": "visualization", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Response Code Timechart", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Response Code Timechart\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"nginx.access.response_code\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Response-Code-Timechart", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Top Remote IPs Table", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Top Remote IPs Table\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.remote_ip\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Top-Remote-IPs-Table", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Map", + "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"nginx.access.geoip.location\"},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"listeners\":{},\"params\":{\"addTooltip\":true,\"heatBlur\":15,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatNormalizeData\":true,\"heatRadius\":25,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[15,5],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"enabled\":false,\"options\":{\"attribution\":\"Maps provided by USGS\",\"format\":\"image/png\",\"layers\":\"0\",\"styles\":\"\",\"transparent\":true,\"version\":\"1.3.0\"},\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"}},\"title\":\"ML Nginx Access Map\",\"type\":\"tile_map\"}" + }, + "id": "ML-Nginx-Access-Map", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Top URLs Table", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Top URLs Table\",\"type\":\"table\",\"params\":{\"perPage\":100,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.url\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Top-URLs-Table", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "columns": [ + "_source" + ], + "description": "Filebeat Nginx Access Data", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:nginx.access\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "ML Nginx Access Data", + "version": 1 + }, + "id": "ML-Filebeat-Nginx-Access", + "type": "search", + "version": 10 + }, + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + }, + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Remote-IP-Timechart\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Response-Code-Timechart\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-Remote-IPs-Table\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Map\",\"col\":7,\"row\":4},{\"size_x\":12,\"size_y\":9,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]", + "timeRestore": false, + "title": "ML Nginx Access Remote IP Count Explorer", + "uiStateJSON": "{\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "version": 1 + }, + "id": "ML-Nginx-Access-Remote-IP-Count-Explorer", + "type": "dashboard", + "version": 1 + } + ], + "version": "6.0.0-alpha3-SNAPSHOT" +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/default/dashboard/ml-nginx-remote-ip-url-explorer.json b/filebeat/module/nginx/_meta/kibana/default/dashboard/ml-nginx-remote-ip-url-explorer.json new file mode 100644 index 00000000000..8336a429e2e --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/default/dashboard/ml-nginx-remote-ip-url-explorer.json @@ -0,0 +1,124 @@ +{ + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Unique Count URL Timechart", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Unique Count URL Timechart\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per day\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of nginx.access.url\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"line\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Unique count of nginx.access.url\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"nginx.access.url\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Unique-Count-URL-Timechart", + "type": "visualization", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Response Code Timechart", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Response Code Timechart\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"nginx.access.response_code\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Response-Code-Timechart", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Top Remote IPs Table", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Top Remote IPs Table\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.remote_ip\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Top-Remote-IPs-Table", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Map", + "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"nginx.access.geoip.location\"},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"listeners\":{},\"params\":{\"addTooltip\":true,\"heatBlur\":15,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatNormalizeData\":true,\"heatRadius\":25,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[15,5],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"enabled\":false,\"options\":{\"attribution\":\"Maps provided by USGS\",\"format\":\"image/png\",\"layers\":\"0\",\"styles\":\"\",\"transparent\":true,\"version\":\"1.3.0\"},\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"}},\"title\":\"ML Nginx Access Map\",\"type\":\"tile_map\"}" + }, + "id": "ML-Nginx-Access-Map", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "savedSearchId": "ML-Filebeat-Nginx-Access", + "title": "ML Nginx Access Top URLs Table", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "visState": "{\"title\":\"ML Nginx Access Top URLs Table\",\"type\":\"table\",\"params\":{\"perPage\":100,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.url\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + }, + "id": "ML-Nginx-Access-Top-URLs-Table", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "columns": [ + "_source" + ], + "description": "Filebeat Nginx Access Data", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:nginx.access\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "ML Nginx Access Data", + "version": 1 + }, + "id": "ML-Filebeat-Nginx-Access", + "type": "search", + "version": 10 + }, + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + }, + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"col\":1,\"id\":\"ML-Nginx-Access-Unique-Count-URL-Timechart\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Nginx-Access-Response-Code-Timechart\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ML-Nginx-Access-Top-Remote-IPs-Table\",\"panelIndex\":3,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Nginx-Access-Map\",\"panelIndex\":4,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":8,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]", + "timeRestore": false, + "title": "ML Nginx Access Remote IP URL Explorer", + "uiStateJSON": "{\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "version": 1 + }, + "id": "ML-Nginx-Remote-IP-URL-Explorer", + "type": "dashboard", + "version": 1 + } + ], + "version": "6.0.0-alpha3-SNAPSHOT" +} \ No newline at end of file diff --git a/filebeat/module/nginx/access/machine_learning/datafeed_low_request_rate.json b/filebeat/module/nginx/access/machine_learning/datafeed_low_request_rate.json new file mode 100644 index 00000000000..776a96ee98e --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/datafeed_low_request_rate.json @@ -0,0 +1,38 @@ +{ + "job_id": "JOB_ID", + "query_delay": "60s", + "frequency": "60s", + "indexes": [ + "filebeat-*" + ], + "types": [ + "_default_", + "log" + ], + "query": { + "match_all": { + "boost": 1 + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "interval": 900000, + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } +} diff --git a/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_request_rate.json b/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_request_rate.json new file mode 100644 index 00000000000..c8d3f09230c --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_request_rate.json @@ -0,0 +1,17 @@ +{ + "job_id": "JOB_ID", + "query_delay": "60s", + "frequency": "60s", + "indexes": [ + "filebeat-*" + ], + "types": [ + "_default_", + "log" + ], + "query": { + "match_all": { + "boost": 1 + } + } +} diff --git a/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_url_count.json b/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_url_count.json new file mode 100644 index 00000000000..c8d3f09230c --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_url_count.json @@ -0,0 +1,17 @@ +{ + "job_id": "JOB_ID", + "query_delay": "60s", + "frequency": "60s", + "indexes": [ + "filebeat-*" + ], + "types": [ + "_default_", + "log" + ], + "query": { + "match_all": { + "boost": 1 + } + } +} diff --git a/filebeat/module/nginx/access/machine_learning/datafeed_response_code.json b/filebeat/module/nginx/access/machine_learning/datafeed_response_code.json index 7231bd0b4fb..c8d3f09230c 100644 --- a/filebeat/module/nginx/access/machine_learning/datafeed_response_code.json +++ b/filebeat/module/nginx/access/machine_learning/datafeed_response_code.json @@ -13,32 +13,5 @@ "match_all": { "boost": 1 } - }, - "aggregations": { - "buckets": { - "date_histogram": { - "field": "@timestamp", - "interval": 3600000, - "offset": 0, - "order": { - "_key": "asc" - }, - "keyed": false, - "min_doc_count": 0 - }, - "aggregations": { - "@timestamp": { - "max": { - "field": "@timestamp" - } - }, - "nginx.access.response_code": { - "terms": { - "field": "nginx.access.response_code", - "size": 10000 - } - } - } - } } } diff --git a/filebeat/module/nginx/access/machine_learning/datafeed_visitor_rate.json b/filebeat/module/nginx/access/machine_learning/datafeed_visitor_rate.json new file mode 100644 index 00000000000..efe05f4d096 --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/datafeed_visitor_rate.json @@ -0,0 +1,43 @@ +{ + "job_id": "JOB_ID", + "query_delay": "60s", + "frequency": "60s", + "indexes": [ + "filebeat-*" + ], + "types": [ + "_default_", + "log" + ], + "query": { + "match_all": { + "boost": 1 + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "interval": 900000, + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_remote_ips": { + "cardinality": { + "field": "nginx.access.remote_ip" + } + } + } + } + } +} diff --git a/filebeat/module/nginx/access/machine_learning/low_request_rate.json b/filebeat/module/nginx/access/machine_learning/low_request_rate.json new file mode 100644 index 00000000000..74fe31b2c66 --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/low_request_rate.json @@ -0,0 +1,30 @@ +{ + "description": "Nginx Access Logs: Detect low request rate", + "analysis_config" : { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "nginx_access_low_request_rate", + "function": "low_count", + "detector_rules": [] + } + ], + "influencers": [] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "custom_urls": [ + { + "url_name": "Raw Data", + "url_value": "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!(),index:\u0027filebeat-*\u0027,interval:auto,query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json b/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json new file mode 100644 index 00000000000..53ea56c264e --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json @@ -0,0 +1,33 @@ +{ + "description": "Nginx Access Logs: Detect unusual remote_ips - high request rates", + "analysis_config" : { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "nginx_access_remote_ip_high_count", + "function": "high_count", + "over_field_name": "nginx.access.remote_ip", + "detector_rules": [] + } + ], + "influencers": [ + "nginx.access.remote_ip" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "custom_urls": [ + { + "url_name": "Count Explorer", + "url_value": "kibana#/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.remote_ip,negate:!f,type:phrase,value:\u0027$nginx.access.remote_ip$\u0027),query:(match:(nginx.access.remote_ip:(query:\u0027$nginx.access.remote_ip$\u0027,type:phrase))))),options:(darkTheme:!f),panels:!((col:1,id:ML-Nginx-Access-Remote-IP-Timechart,panelIndex:1,row:1,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Response-Code-Timechart,panelIndex:2,row:1,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-Remote-IPs-Table,panelIndex:3,row:4,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Map,panelIndex:4,row:4,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-URLs-Table,panelIndex:5,row:7,size_x:12,size_y:9,type:visualization)),query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),timeRestore:!f,title:\u0027ML%20Nginx%20Access%20Remote%20IP%20Count%20Explorer\u0027,uiState:(P-3:(vis:(params:(sort:(columnIndex:!n,direction:!n)))),P-5:(vis:(params:(sort:(columnIndex:!n,direction:!n))))),viewMode:view)" + }, + { + "url_name": "Raw Data", + "url_value": "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.remote_ip,negate:!f,type:phrase,value:\u0027$nginx.access.remote_ip$\u0027),query:(match:(nginx.access.remote_ip:(query:\u0027$nginx.access.remote_ip$\u0027,type:phrase))))),index:\u0027filebeat-*\u0027,interval:auto,query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json b/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json new file mode 100644 index 00000000000..28ccd5a1937 --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json @@ -0,0 +1,34 @@ +{ + "description": "Nginx Access Logs: Detect unusual remote_ips - high distinct count of urls", + "analysis_config" : { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "nginx_access_remote_ip_high_dc_url", + "function": "high_distinct_count", + "field_name": "nginx.access.url", + "over_field_name": "nginx.access.remote_ip", + "detector_rules": [] + } + ], + "influencers": [ + "nginx.access.remote_ip" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "custom_urls": [ + { + "url_name": "URL Explorer", + "url_value": "kibana#/dashboard/ML-Nginx-Remote-IP-URL-Explorer?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.remote_ip,negate:!f,type:phrase,value:\u0027$nginx.access.remote_ip$\u0027),query:(match:(nginx.access.remote_ip:(query:\u0027$nginx.access.remote_ip$\u0027,type:phrase))))),options:(darkTheme:!f),panels:!((col:1,id:ML-Nginx-Access-Unique-Count-URL-Timechart,panelIndex:1,row:1,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Response-Code-Timechart,panelIndex:2,row:1,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-Remote-IPs-Table,panelIndex:3,row:4,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Map,panelIndex:4,row:4,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-URLs-Table,panelIndex:5,row:7,size_x:12,size_y:8,type:visualization)),query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),timeRestore:!f,title:\u0027ML%20Nginx%20Access%20Remote%20IP%20URL%20Explorer\u0027,uiState:(P-2:(vis:(params:(sort:(columnIndex:!n,direction:!n)))),P-3:(vis:(params:(sort:(columnIndex:!n,direction:!n)))),P-5:(vis:(params:(sort:(columnIndex:!n,direction:!n))))),viewMode:view)" + }, + { + "url_name": "Raw Data", + "url_value": "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.remote_ip,negate:!f,type:phrase,value:\u0027$nginx.access.remote_ip$\u0027),query:(match:(nginx.access.remote_ip:(query:\u0027$nginx.access.remote_ip$\u0027,type:phrase))))),index:\u0027filebeat-*\u0027,interval:auto,query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/filebeat/module/nginx/access/machine_learning/response_code.json b/filebeat/module/nginx/access/machine_learning/response_code.json index efb0b2274d1..ee4a71cddf1 100644 --- a/filebeat/module/nginx/access/machine_learning/response_code.json +++ b/filebeat/module/nginx/access/machine_learning/response_code.json @@ -1,17 +1,19 @@ { - "description" : "Anomaly detector for changes in event rates of nginx.access.response_code responses", + "description": "Nginx Access Logs: Detect unusual response_code rates", "analysis_config" : { - "bucket_span": "1h", - "summary_count_field_name": "doc_count", + "bucket_span": "15m", "detectors": [ - { - "detector_description": "Event rate for nginx.access.response_code", + { + "detector_description": "nginx_access_response_code_rate", "function": "count", - "detector_rules": [], - "partition_field_name": "nginx.access.response_code" + "partition_field_name": "nginx.access.response_code", + "detector_rules": [] } ], - "influencers": ["nginx.access.response_code"] + "influencers": [ + "nginx.access.response_code", + "nginx.access.remote_ip" + ] }, "data_description": { "time_field": "@timestamp", @@ -19,5 +21,17 @@ }, "model_plot_config": { "enabled": true + }, + "custom_settings": { + "custom_urls": [ + { + "url_name": "Count Explorer", + "url_value": "kibana#/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.response_code,negate:!f,type:phrase,value:\u0027$nginx.access.response_code$\u0027),query:(match:(nginx.access.response_code:(query:\u0027$nginx.access.response_code$\u0027,type:phrase))))),options:(darkTheme:!f),panels:!((col:1,id:ML-Nginx-Access-Remote-IP-Timechart,panelIndex:1,row:1,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Response-Code-Timechart,panelIndex:2,row:1,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-Remote-IPs-Table,panelIndex:3,row:4,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Map,panelIndex:4,row:4,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-URLs-Table,panelIndex:5,row:7,size_x:12,size_y:9,type:visualization)),query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),timeRestore:!f,title:\u0027ML%20Nginx%20Access%20Remote%20IP%20Count%20Explorer\u0027,uiState:(P-3:(vis:(params:(sort:(columnIndex:!n,direction:!n)))),P-5:(vis:(params:(sort:(columnIndex:!n,direction:!n))))),viewMode:view)" + }, + { + "url_name": "Raw Data", + "url_value": "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.response_code,negate:!f,type:phrase,value:\u0027$nginx.access.response_code$\u0027),query:(match:(nginx.access.response_code:(query:\u0027$nginx.access.response_code$\u0027,type:phrase))))),index:\u0027filebeat-*\u0027,interval:auto,query:(query_string:(analyze_wildcard:!t,query:\u0027_exists_:nginx.access\u0027)),sort:!(\u0027@timestamp\u0027,desc))" + } + ] } } diff --git a/filebeat/module/nginx/access/machine_learning/visitor_rate.json b/filebeat/module/nginx/access/machine_learning/visitor_rate.json new file mode 100644 index 00000000000..bc5341e55b8 --- /dev/null +++ b/filebeat/module/nginx/access/machine_learning/visitor_rate.json @@ -0,0 +1,30 @@ +{ + "description": "Nginx Access Logs: Detect unusual visitor rate", + "analysis_config" : { + "bucket_span": "15m", + "summary_count_field_name": "dc_remote_ips", + "detectors": [ + { + "detector_description": "nginx_access_visitor_rate", + "function": "non_zero_count", + "detector_rules": [] + } + ], + "influencers": [] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "custom_urls": [ + { + "url_name": "Raw Data", + "url_value": "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!(),index:\u0027filebeat-*\u0027,interval:auto,query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/filebeat/module/nginx/access/manifest.yml b/filebeat/module/nginx/access/manifest.yml index 69dc9aab323..c04eac7ddfc 100644 --- a/filebeat/module/nginx/access/manifest.yml +++ b/filebeat/module/nginx/access/manifest.yml @@ -16,6 +16,18 @@ machine_learning: - name: response_code job: machine_learning/response_code.json datafeed: machine_learning/datafeed_response_code.json +- name: low_request_rate + job: machine_learning/low_request_rate.json + datafeed: machine_learning/datafeed_low_request_rate.json +- name: remote_ip_url_count + job: machine_learning/remote_ip_url_count.json + datafeed: machine_learning/datafeed_remote_ip_url_count.json +- name: remote_ip_request_rate + job: machine_learning/remote_ip_request_rate.json + datafeed: machine_learning/datafeed_remote_ip_request_rate.json +- name: visitor_rate + job: machine_learning/visitor_rate.json + datafeed: machine_learning/datafeed_visitor_rate.json requires.processors: - name: user_agent diff --git a/testing/environments/docker/kibana/Dockerfile-snapshot b/testing/environments/docker/kibana/Dockerfile-snapshot index 8a01f46b23a..404cb2c3c81 100644 --- a/testing/environments/docker/kibana/Dockerfile-snapshot +++ b/testing/environments/docker/kibana/Dockerfile-snapshot @@ -14,7 +14,7 @@ RUN curl -Ls ${DOWNLOAD_URL}/kibana/kibana-${ELASTIC_VERSION}-linux-x86_64.tar.g ln -s /usr/share/kibana /opt/kibana # Install XPACK -RUN if [ ${XPACK} = 1]; then bin/kibana-plugin install ${DOWNLOAD_URL}/kibana-plugins/x-pack/x-pack-${ELASTIC_VERSION}.zip; fi +RUN if [ ${XPACK} = "1" ]; then bin/kibana-plugin install ${DOWNLOAD_URL}/kibana-plugins/x-pack/x-pack-${ELASTIC_VERSION}.zip; fi # Set some Kibana configuration defaults. ADD config/kibana.yml /usr/share/kibana/config/ diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 9a87fbc0d6b..df9d5566c2a 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -25,8 +25,6 @@ services: build: context: ./docker/logstash dockerfile: Dockerfile - args: - XPACK: 1 environment: - ES_HOST=elasticsearch @@ -37,3 +35,5 @@ services: build: context: ./docker/kibana dockerfile: Dockerfile-snapshot + args: + XPACK: 1