From a03d6db435087e2c39d2a3f9a789407361b0033d Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Fri, 22 Feb 2019 18:39:41 -0800 Subject: [PATCH 01/16] Update Zeek dashboard --- .../7/dashboard/Filebeat-Zeek-Overview.json | 650 +++++++++++------- x-pack/filebeat/module/zeek/module.yml | 2 +- 2 files changed, 402 insertions(+), 250 deletions(-) diff --git a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json index 5fd7816eb982..b8d9553330fd 100644 --- a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json +++ b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json @@ -3,17 +3,194 @@ { "attributes": { "description": "", + "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Destination Geo [SIEM Zeek] ECS", + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 20, + "i": "1", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "2", + "w": 16, + "x": 0, + "y": 20 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "3", + "w": 16, + "x": 16, + "y": 20 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "4", + "w": 16, + "x": 32, + "y": 20 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "5", + "w": 16, + "x": 0, + "y": 32 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "6", + "w": 16, + "x": 16, + "y": 32 + }, + "panelIndex": "6", + "panelRefName": "panel_5", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "7", + "w": 16, + "x": 32, + "y": 32 + }, + "panelIndex": "7", + "panelRefName": "panel_6", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "8", + "w": 48, + "x": 0, + "y": 44 + }, + "panelIndex": "8", + "panelRefName": "panel_7", + "version": "7.0.0-beta1" + } + ], + "timeRestore": false, + "title": "Zeek Overview Dashboard", + "version": 1 + }, + "id": "7cbb5410-3700-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "dashboard": "7.0.0" + }, + "references": [ + { + "id": "f469f230-370c-11e9-aa6d-ff445a78330c", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "1df7ea80-370d-11e9-aa6d-ff445a78330c", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "466e5850-370d-11e9-aa6d-ff445a78330c", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "649acd40-370d-11e9-aa6d-ff445a78330c", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "9436c270-370d-11e9-aa6d-ff445a78330c", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "bec2f0e0-370d-11e9-aa6d-ff445a78330c", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "e042fda0-370d-11e9-aa6d-ff445a78330c", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "f8c40810-370d-11e9-aa6d-ff445a78330c", + "name": "panel_7", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2019-02-23T01:54:58.557Z", + "version": "WzE5MjksNF0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Destination Geo [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -47,6 +224,18 @@ "params": { "addTooltip": true, "colorSchema": "Yellow to Red", + "dimensions": { + "geocentroid": null, + "geohash": null, + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "heatClusterSize": 1.5, "isDesaturated": true, "legendPosition": "bottomright", @@ -61,25 +250,27 @@ "options": { "format": "image/png", "transparent": true - }, - "selectedTmsLayer": { - "attribution": "\u003cp\u003e\u0026#169; \u003ca href=\"http://www.openstreetmap.org/copyright\"\u003eOpenStreetMap\u003c/a\u003e contributors | \u003ca href=\"https://www.elastic.co/elastic-maps-service\"\u003eElastic Maps Service\u003c/a\u003e\u003c/p\u003e\u0026#10;", - "id": "road_map", - "maxZoom": 18, - "minZoom": 0, - "subdomains": [], - "url": "https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree\u0026my_app_name=kibana\u0026my_app_version=6.5.4\u0026license=decdfd78-7d5b-47b7-9627-603d9b789d29" } } }, - "title": "Destination Geo [SIEM Zeek] ECS", + "title": "Destination Geo [SIEM Zeek]", "type": "tile_map" } }, - "id": "5d95a3e0-1a29-11e9-84b1-a12c578fa9e8-ecs", + "id": "f469f230-370c-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:27:37.758Z", - "version": 1 + "updated_at": "2019-02-23T01:47:19.123Z", + "version": "WzE5MjEsNF0=" }, { "attributes": { @@ -87,14 +278,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Network Transport [SIEM Zeek] ECS", + "title": "Network Transport [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -126,6 +317,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -136,14 +337,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Transport [SIEM Zeek] ECS", + "title": "Network Transport [SIEM Zeek]", "type": "pie" } }, - "id": "c337dbf0-1a29-11e9-84b1-a12c578fa9e8-ecs", + "id": "1df7ea80-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:30:28.271Z", - "version": 1 + "updated_at": "2019-02-23T01:48:28.840Z", + "version": "WzE5MjIsNF0=" }, { "attributes": { @@ -151,14 +362,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Network Application [SIEM Zeek] ECS", + "title": "Network Application [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -181,7 +392,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 10 + "size": 5 }, "schema": "segment", "type": "terms" @@ -190,6 +401,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -200,14 +421,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Application [SIEM Zeek] ECS", + "title": "Network Application [SIEM Zeek]", "type": "pie" } }, - "id": "f054ee70-1a29-11e9-84b1-a12c578fa9e8-ecs", + "id": "466e5850-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:31:43.959Z", - "version": 1 + "updated_at": "2019-02-23T01:49:36.725Z", + "version": "WzE5MjMsNF0=" }, { "attributes": { @@ -215,14 +446,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Network Traffic Direction [SIEM Zeek] ECS", + "title": "Network Traffic Direction [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -254,6 +485,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -264,14 +505,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Traffic Direction [SIEM Zeek] ECS", + "title": "Network Traffic Direction [SIEM Zeek]", "type": "pie" } }, - "id": "15922a40-1a2a-11e9-84b1-a12c578fa9e8-ecs", + "id": "649acd40-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:32:46.436Z", - "version": 1 + "updated_at": "2019-02-23T01:50:27.347Z", + "version": "WzE5MjQsNF0=" }, { "attributes": { @@ -279,14 +530,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Top DNS Domains [SIEM Zeek] ECS", + "title": "Top DNS Domains [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -309,7 +560,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 8 + "size": 10 }, "schema": "segment", "type": "terms" @@ -318,6 +569,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -328,14 +589,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top DNS Domains [SIEM Zeek] ECS", + "title": "Top DNS Domains [SIEM Zeek]", "type": "pie" } }, - "id": "b3705f00-1a2c-11e9-84b1-a12c578fa9e8-ecs", + "id": "9436c270-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:51:30.288Z", - "version": 1 + "updated_at": "2019-02-23T01:51:47.223Z", + "version": "WzE5MjUsNF0=" }, { "attributes": { @@ -343,14 +614,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Top URL Domain [SIEM Zeek] ECS", + "title": "Top URL Domains [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -373,7 +644,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 8 + "size": 10 }, "schema": "segment", "type": "terms" @@ -382,6 +653,31 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "buckets": [ + { + "accessor": 0, + "aggType": "terms", + "format": { + "id": "terms", + "params": { + "id": "string", + "missingBucketLabel": "Missing", + "otherBucketLabel": "Other" + } + }, + "params": {} + } + ], + "metric": { + "accessor": 1, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -392,14 +688,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top URL Domain [SIEM Zeek] ECS", + "title": "Top URL Domains [SIEM Zeek]", "type": "pie" } }, - "id": "ef0cfdc0-1a2c-11e9-84b1-a12c578fa9e8-ecs", + "id": "bec2f0e0-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:53:10.300Z", - "version": 1 + "updated_at": "2019-02-23T01:52:58.606Z", + "version": "WzE5MjYsNF0=" }, { "attributes": { @@ -407,14 +713,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Top SSL Server [SIEM Zeek] ECS", + "title": "Top SSL Servers [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -437,7 +743,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 8 + "size": 10 }, "schema": "segment", "type": "terms" @@ -446,6 +752,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -456,14 +772,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top SSL Server [SIEM Zeek] ECS", + "title": "Top SSL Servers [SIEM Zeek]", "type": "pie" } }, - "id": "13454cb0-1a2d-11e9-84b1-a12c578fa9e8-ecs", + "id": "e042fda0-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:54:11.067Z", - "version": 1 + "updated_at": "2019-02-23T01:53:54.810Z", + "version": "WzE5MjcsNF0=" }, { "attributes": { @@ -472,12 +798,12 @@ "searchSourceJSON": { "filter": [], "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Time Series Count [SIEM Zeek] ECS", + "title": "Time Series Count [SIEM Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -486,26 +812,8 @@ "axis_formatter": "number", "axis_position": "left", "axis_scale": "normal", - "background_color_rules": [ - { - "id": "3716ea90-1a2d-11e9-b2af-13b289f0bf65" - } - ], - "bar_color_rules": [ - { - "id": "3822dc50-1a2d-11e9-b2af-13b289f0bf65" - } - ], - "gauge_color_rules": [ - { - "id": "4c1a3ff0-1a2d-11e9-b2af-13b289f0bf65" - } - ], - "gauge_inner_width": 10, - "gauge_style": "half", - "gauge_width": 10, "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "filebeat-*", + "index_pattern": "", "interval": "auto", "series": [ { @@ -513,7 +821,6 @@ "chart_type": "line", "color": "#68BC00", "fill": 0.5, - "filter": "tags:zeek", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", "line_width": 1, @@ -525,183 +832,28 @@ ], "point_size": 1, "separate_axis": 0, - "split_mode": "filter", + "split_mode": "everything", "stacked": "none" } ], "show_grid": 1, - "show_legend": 0, + "show_legend": 1, "time_field": "@timestamp", "type": "timeseries" }, - "title": "Time Series Count [SIEM Zeek] ECS", + "title": "Time Series Count [SIEM Zeek]", "type": "metrics" } }, - "id": "fad258c0-1078-11e9-b27a-69e6e8b80a25-ecs", - "type": "visualization", - "updated_at": "2019-01-17T07:56:26.486Z", - "version": 74 - }, - { - "attributes": { - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "lucene", - "query": "" - } - } - }, - "optionsJSON": { - "darkTheme": false, - "hidePanelTitles": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": { - "mapCenter": [ - 20.3034175184893, - -5.537109375000001 - ], - "mapZoom": 2 - }, - "gridData": { - "h": 18, - "i": "1", - "w": 48, - "x": 0, - "y": 0 - }, - "id": "5d95a3e0-1a29-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "1", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": true - } - }, - "gridData": { - "h": 10, - "i": "2", - "w": 16, - "x": 0, - "y": 18 - }, - "id": "c337dbf0-1a29-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "2", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": true - } - }, - "gridData": { - "h": 10, - "i": "3", - "w": 17, - "x": 16, - "y": 18 - }, - "id": "f054ee70-1a29-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "3", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": true - } - }, - "gridData": { - "h": 10, - "i": "4", - "w": 15, - "x": 33, - "y": 18 - }, - "id": "15922a40-1a2a-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "4", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "5", - "w": 16, - "x": 0, - "y": 28 - }, - "id": "b3705f00-1a2c-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "5", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "6", - "w": 17, - "x": 16, - "y": 28 - }, - "id": "ef0cfdc0-1a2c-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "6", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "7", - "w": 15, - "x": 33, - "y": 28 - }, - "id": "13454cb0-1a2d-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "7", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 9, - "i": "8", - "w": 48, - "x": 0, - "y": 39 - }, - "id": "fad258c0-1078-11e9-b27a-69e6e8b80a25-ecs", - "panelIndex": "8", - "type": "visualization", - "version": "6.5.4" - } - ], - "timeRestore": false, - "title": "Zeek Overview Dashboard [SIEM] ECS", - "version": 1 + "id": "f8c40810-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" }, - "id": "87b0c430-1a2d-11e9-84b1-a12c578fa9e8-ecs", - "type": "dashboard", - "updated_at": "2019-01-17T07:57:50.613Z", - "version": 2 + "references": [], + "type": "visualization", + "updated_at": "2019-02-23T01:54:35.921Z", + "version": "WzE5MjgsNF0=" } ], - "version": "6.5.4" + "version": "7.0.0-beta1" } \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/module.yml b/x-pack/filebeat/module/zeek/module.yml index 0db598900870..4d55536c0c6f 100644 --- a/x-pack/filebeat/module/zeek/module.yml +++ b/x-pack/filebeat/module/zeek/module.yml @@ -1,3 +1,3 @@ dashboards: -- id: 87b0c430-1a2d-11e9-84b1-a12c578fa9e8 +- id: 7cbb5410-3700-11e9-aa6d-ff445a78330c file: Filebeat-Zeek-Overview.json From c356dc2c17a32ce2aa17ee948abe8d864a8f0f19 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Fri, 22 Feb 2019 20:27:26 -0800 Subject: [PATCH 02/16] Update README.md --- x-pack/filebeat/module/zeek/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/zeek/README.md b/x-pack/filebeat/module/zeek/README.md index 44a51dbf456a..54e7e568ad5f 100644 --- a/x-pack/filebeat/module/zeek/README.md +++ b/x-pack/filebeat/module/zeek/README.md @@ -44,7 +44,7 @@ Grab the filebeat binary from elastic.co, and install it by following the instru Update filebeat.yml to point to Elasticsearch and Kibana. Setup Filebeat. ``` -./filebeat setup --modules zeek -e -E setup.dashboards.directory=build/kibana +./filebeat setup --modules zeek -e -E 'setup.dashboards.enabled=true' ``` Enable the Filebeat zeek module From f5fbd248b44ecd696b0b30601bd54a3e56a59455 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Sat, 23 Feb 2019 09:37:54 -0800 Subject: [PATCH 03/16] Fix a typo in README - network.cfg should be networks.cfg --- x-pack/filebeat/module/zeek/README-developer.md | 4 ++-- x-pack/filebeat/module/zeek/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/module/zeek/README-developer.md b/x-pack/filebeat/module/zeek/README-developer.md index a1b431b64a6c..c57cb873c11a 100644 --- a/x-pack/filebeat/module/zeek/README-developer.md +++ b/x-pack/filebeat/module/zeek/README-developer.md @@ -14,7 +14,7 @@ brew install bro * Configure it to process network traffic and generate logs. * Edit `/usr/local/etc/node.cfg` to use the proper network interfaces. -* Edit `/usr/local/etc/network.cfg` to specify local networks accordingly. +* Edit `/usr/local/etc/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/local/share/bro/site/local.bro` to use JSON output. ### Install Zeek/Bro (for Ubuntu Linux) @@ -52,7 +52,7 @@ mage build Update filebeat.yml to point to Elasticsearch and Kibana. Setup Filebeat. ``` -./filebeat setup --modules zeek -e -E setup.dashboards.directory=build/kibana +./filebeat setup --modules zeek -e -E 'setup.dashboards.directory=build/kibana' ``` Enable the Filebeat zeek module diff --git a/x-pack/filebeat/module/zeek/README.md b/x-pack/filebeat/module/zeek/README.md index 54e7e568ad5f..7dab73713eb7 100644 --- a/x-pack/filebeat/module/zeek/README.md +++ b/x-pack/filebeat/module/zeek/README.md @@ -26,7 +26,7 @@ apt install broctl * Configure it to process network traffic and generate logs. * Edit `/etc/bro/node.cfg` to use the proper network interfaces. -* Edit `/etc/bro/network.cfg` to specify local networks accordingly. +* Edit `/etc/bro/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/share/bro/site/local.bro` to use JSON output. ## Start Zeek/Bro From 1fca544b9ec8d700dd6c30e7d2e78110ee4a0547 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Sat, 23 Feb 2019 09:45:31 -0800 Subject: [PATCH 04/16] Fix a typo in README - network.cfg should be networks.cfg --- x-pack/filebeat/module/zeek/README-developer.md | 2 +- x-pack/filebeat/module/zeek/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/module/zeek/README-developer.md b/x-pack/filebeat/module/zeek/README-developer.md index c57cb873c11a..20410f14c1bf 100644 --- a/x-pack/filebeat/module/zeek/README-developer.md +++ b/x-pack/filebeat/module/zeek/README-developer.md @@ -26,7 +26,7 @@ apt install broctl * Configure it to process network traffic and generate logs. * Edit `/etc/bro/node.cfg` to use the proper network interfaces. -* Edit `/etc/bro/network.cfg` to specify local networks accordingly. +* Edit `/etc/bro/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/share/bro/site/local.bro` to use JSON output. ## Start Zeek/Bro diff --git a/x-pack/filebeat/module/zeek/README.md b/x-pack/filebeat/module/zeek/README.md index 7dab73713eb7..740fff626412 100644 --- a/x-pack/filebeat/module/zeek/README.md +++ b/x-pack/filebeat/module/zeek/README.md @@ -14,7 +14,7 @@ brew install bro * Configure it to process network traffic and generate logs. * Edit `/usr/local/etc/node.cfg` to use the proper network interfaces. -* Edit `/usr/local/etc/network.cfg` to specify local networks accordingly. +* Edit `/usr/local/etc/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/local/share/bro/site/local.bro` to use JSON output. ### Install Zeek/Bro (for Ubuntu Linux) From a3061ac70adad75b7be295d19e5a91d5bcca6fae Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Sat, 23 Feb 2019 15:09:05 -0800 Subject: [PATCH 05/16] Add support for notice.log --- filebeat/docs/fields.asciidoc | 154 ++++++++++++++++++ x-pack/filebeat/filebeat.reference.yml | 2 + x-pack/filebeat/module/zeek/_meta/config.yml | 2 + x-pack/filebeat/module/zeek/_meta/fields.yml | 68 ++++++++ .../module/zeek/connection/manifest.yml | 2 +- .../test/connection-json.log-expected.json | 2 +- x-pack/filebeat/module/zeek/dns/manifest.yml | 2 +- .../zeek/dns/test/dns-json.log-expected.json | 2 +- x-pack/filebeat/module/zeek/fields.go | 2 +- .../filebeat/module/zeek/files/manifest.yml | 2 +- .../files/test/files-json.log-expected.json | 4 +- x-pack/filebeat/module/zeek/http/manifest.yml | 2 +- .../http/test/http-json.log-expected.json | 2 +- .../module/zeek/notice/config/notice.yml | 113 +++++++++++++ .../module/zeek/notice/ingest/pipeline.json | 47 ++++++ .../filebeat/module/zeek/notice/manifest.yml | 19 +++ .../module/zeek/notice/test/notice-json.log | 1 + .../notice/test/notice-json.log-expected.json | 24 +++ x-pack/filebeat/module/zeek/ssl/manifest.yml | 2 +- .../zeek/ssl/test/ssl-json.log-expected.json | 4 +- x-pack/filebeat/modules.d/zeek.yml.disabled | 2 + 21 files changed, 445 insertions(+), 13 deletions(-) create mode 100644 x-pack/filebeat/module/zeek/notice/config/notice.yml create mode 100644 x-pack/filebeat/module/zeek/notice/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/zeek/notice/manifest.yml create mode 100644 x-pack/filebeat/module/zeek/notice/test/notice-json.log create mode 100644 x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 6df8c5610567..c454f1766eef 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -14112,3 +14112,157 @@ type: keyword -- +*`zeek.notice.connection_id`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.icmp_id`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.file.id`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.file.parent_id`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.file.source`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.file.is_orig`*:: ++ +-- +type: boolean + +-- + +*`zeek.notice.file.seen_bytes`*:: ++ +-- +type: long + +-- + +*`zeek.notice.file.total_bytes`*:: ++ +-- +type: long + +-- + +*`zeek.notice.fuid`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.file_mime_type`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.note`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.msg`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.sub`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.n`*:: ++ +-- +type: long + +-- + +*`zeek.notice.peer_name`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.peer_descr`*:: ++ +-- +type: text + +-- + +*`zeek.notice.actions`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.email_body_sections`*:: ++ +-- +type: text + +-- + +*`zeek.notice.email_delay_tokens`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.identifier`*:: ++ +-- +type: keyword + +-- + +*`zeek.notice.suppress_for`*:: ++ +-- +type: double + +-- + +*`zeek.notice.dropped`*:: ++ +-- +type: boolean + +-- + diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 8d41832ce472..1dc451b88d13 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -437,6 +437,8 @@ filebeat.modules: enabled: true ssl: enabled: true + notice: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/x-pack/filebeat/module/zeek/_meta/config.yml b/x-pack/filebeat/module/zeek/_meta/config.yml index a79fc0456c29..22bf8b09f276 100644 --- a/x-pack/filebeat/module/zeek/_meta/config.yml +++ b/x-pack/filebeat/module/zeek/_meta/config.yml @@ -10,6 +10,8 @@ enabled: true ssl: enabled: true + notice: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/x-pack/filebeat/module/zeek/_meta/fields.yml b/x-pack/filebeat/module/zeek/_meta/fields.yml index 60c59f4e75d8..912e86ef7dcb 100644 --- a/x-pack/filebeat/module/zeek/_meta/fields.yml +++ b/x-pack/filebeat/module/zeek/_meta/fields.yml @@ -284,4 +284,72 @@ - name: ssl.last_alert type: keyword + - name: notice.connection_id + type: keyword + + - name: notice.icmp_id + type: keyword + + - name: notice.file.id + type: keyword + + - name: notice.file.parent_id + type: keyword + + - name: notice.file.source + type: keyword + + - name: notice.file.is_orig + type: boolean + + - name: notice.file.seen_bytes + type: long + + - name: notice.file.total_bytes + type: long + + - name: notice.fuid + type: keyword + + - name: notice.file_mime_type + type: keyword + + - name: notice.note + type: keyword + + - name: notice.msg + type: keyword + + - name: notice.sub + type: keyword + + - name: notice.n + type: long + + - name: notice.peer_name + type: keyword + + - name: notice.peer_descr + type: text + + - name: notice.actions + type: keyword + + - name: notice.email_body_sections + type: text + + - name: notice.email_delay_tokens + type: keyword + + - name: notice.identifier + type: keyword + + - name: notice.suppress_for + type: double + + - name: notice.dropped + type: boolean + + + diff --git a/x-pack/filebeat/module/zeek/connection/manifest.yml b/x-pack/filebeat/module/zeek/connection/manifest.yml index 53e7f507cd66..fc71598ebddd 100644 --- a/x-pack/filebeat/module/zeek/connection/manifest.yml +++ b/x-pack/filebeat/module/zeek/connection/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/conn.log - name: tags - default: [zeek] + default: [zeek.connection] ingest_pipeline: ingest/pipeline.json input: config/connection.yml diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json index 89b37e6e83e4..dc745da83a12 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json @@ -24,7 +24,7 @@ "source.packets": 1, "source.port": 38339, "tags": [ - "zeek" + "zeek.connection" ], "zeek.connection.history": "Dd", "zeek.connection.local_orig": true, diff --git a/x-pack/filebeat/module/zeek/dns/manifest.yml b/x-pack/filebeat/module/zeek/dns/manifest.yml index da306cc5cfe5..71032e045d87 100644 --- a/x-pack/filebeat/module/zeek/dns/manifest.yml +++ b/x-pack/filebeat/module/zeek/dns/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/dns.log - name: tags - default: [zeek] + default: [zeek.dns] ingest_pipeline: ingest/pipeline.json input: config/dns.yml diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json index f30c13cfaf6e..4a273973b425 100644 --- a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json @@ -17,7 +17,7 @@ "source.ip": "192.168.86.167", "source.port": 38339, "tags": [ - "zeek" + "zeek.dns" ], "zeek.dns.AA": false, "zeek.dns.RA": true, diff --git a/x-pack/filebeat/module/zeek/fields.go b/x-pack/filebeat/module/zeek/fields.go index fe6e78a484e1..bc440ae8bff3 100644 --- a/x-pack/filebeat/module/zeek/fields.go +++ b/x-pack/filebeat/module/zeek/fields.go @@ -19,5 +19,5 @@ func init() { // AssetZeek returns asset data. // This is the base64 encoded gzipped contents of module/zeek. func AssetZeek() string { - return "eJycl7GOIzcMhns/hV7gNsgBl8JFgEuCVEkTbJVGkCXOjLIaUUdK3vU+fSCNY2M99g2V0vb3kxRFivQn9QKnvXoHeNkplX0OsFd/L58csCWfsse4Vz/vlFLqT3QlgBqQ1GSiCz6OKuDIKhG6YsGpw6nJf/iFcKfU4CE43u+U+qSimeHiSal8SrBXI2FJ7fMdb0r93vRqIJwvZheHZshAKiLNJvh3U2VNc/V49cnA7DFq785mF98vcHpFcrsPrMUYwVZzTwGtCRrJjx9kB8QAJm7ICDh1yWbPDE4fThn4gzBgHB+qOJsMXaeaPGekU5em5kCHz9o4R9vCenQ5fQwmblM+RiAtY13kp0wm8u2F+5hhBFrTlPMH0GE5BFhz3wpIMtdQGwxvXeQV1PULoeX6u8Rw/bHDLll0EruN67D79et2H1Tu+VcZ99dvQk7o10R+BWLZYZ6f/2BRqRD8AzaDuxtD/WZdsphN0PeieVi3i4QgBQ9CCZtXvS7jh8mpeLUvwKec07nvHKQ83Y1ndfKmqq9YYT3zePcW7ot8HFCvavb7jprmkZs1PvgAskJfDm9GQRk11JqUC4HTyTBfDrmd30T45kEwxxpNJo6gCb4V4Cz0YIOHmPUExgG1NpeeiYGOQP9H2ebLULzrEsx+Bl3JPjfnO5WK2jTria0JumNb3HTGdk3D4477XoBCVQ2Ln2oOtuNa2PymJ7wpOZ/ukSQmr1ucID1nCRayoKT4Oh23E5AvqKdlN5LaNtGE07toziyCSwVJBfLHauF71ttF4bkLd4XMKkX35uV/9wtRtP6eS6zNPjlf92sfxw4FHoGGgK89QfkZHBbBW7vwyVB9bHtaa3ZfxOU/mR872M9ffpLS8JbJrNYbiUDbknEYpAm66ti/b22nZ0nMhOm0WXXM4ekIxKImrrD1aQLBP5rGFjoKOrGi57Epa90qIOAyP9gsV3CEt6wTYUaLQWYfOJtD8DxJfVigrO1kvDSRF146WJtqWUy6nd3Kenx65iK+8sVRj+RognftjdTLCiysmHKofy66wurSBMNZmwD0gN/9GwAA///PiHUF" + return "eJycmMFy4zYMhu9+Cr7AutOd2R5y6My2nZ7aSyenXji0CElsKIILUE6Up++Q0tpxJEfgHm39HwGCAAHpk3qC6UG9AjwdlEoueXhQ/86/LHBDLiaH4UH9elBKqb/Rjh5Ui6R6E6x3oVMeO1aR0I4NWHWaCv7Tb4QHpVoH3vLDQalPKpgBLpaUSlOEB9URjrH83rCm1J+FVy3hcFl2NmjaBKQC0mC8ezUZK8zV4tUmA7PDoJ1dlp1tP8H0jGQPN9oGQ4AmL3f02BivkVx3g50QPZiwgxFwrMIGxwxWn6YEfAN6DN1dipNJULWr3nFCmqqYHAPtP2tjLe2Deety9dmbsK9yIQBpmdYGPiYygd8fuAsJOqC1mlK6EVocTx7Wum8jSCJXpI03vHeQV6HOfwhXzs8lC+eHFetSg1aybtFVrPv1634dZN3j7zLdP38IdUK7JvAzEMs28/j4F4tSheA/aBLYTR/yP+uUxWS83vLmbt7OCEH0DoQIm2e9TuO7wcnyvL5A3qcUl7qzEFO/6c9q54XKt9jIeuBu8xS2IRda1Kuc/dhQYe6ZWctb50GW6PPmTSdIoyJtTEwjgdXRMF82uR/fSPjiQNDHippM6EATfBuBk9BC4x2EpHswFqiUuXRPDHQG+hGy9Jd2dLYKGNwAOivrzCxnKoVKN6vxrQDVvs1mKn27huF+xX3koJDKbvExx2Dfr1mbXnSP71LOxS0liZXXKU4QngXBkRpQUvk6HO87IF+kjubZSLq2CcZPr6I+MwOXDJIC8stq1teMtzPhuEpuRzKrEG31y+/nC0E0/i4pVnqfXJ/naxe6CgLPQK3H5xqn3AAWR8FdO+ujoXzZ1pTWYL+I0783P1doP3/5RaqGl0RmNd5IAN2MCdtWGqArx+51bzpdkJAI47Sbdcz+eAZiURFnceNiD4I3mqId6SyoxCxd2qasdDNAwONwZ7JciQO8JB0JEzboZesDJ3PyjnupjQYo6aY3ThrIi17aWAs1DybVxt5jNTYd8yg+8tlQDXI23tlyR+p5BBZmzHjKLxdVblUx3nDSxgMJ9AGTa+B4/Vog+syyUK4ZYo0+V/ixVr9csrXYPDxUuibtkDeGpI3vLSRvf98pUY95Y0JXTCALFlDyMWoRi17DFi2PJ7kTsnhEEF+6b4nymfIGSfCSNvWmlIOgphc9DMZ5fUI7aYYN9q6hGbTgzaQTPkGFTWchJNc6yY11OYsYCZh1i7TbZBfGEsb4UTs5HA7/BwAA//9bGsLO" } diff --git a/x-pack/filebeat/module/zeek/files/manifest.yml b/x-pack/filebeat/module/zeek/files/manifest.yml index 9da593ea2ed6..1d9ac2207610 100644 --- a/x-pack/filebeat/module/zeek/files/manifest.yml +++ b/x-pack/filebeat/module/zeek/files/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/files.log - name: tags - default: [zeek] + default: [zeek.files] ingest_pipeline: ingest/pipeline.json input: config/files.yml diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json index c5d2d872e2f9..f2c9f37d4602 100644 --- a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json @@ -10,7 +10,7 @@ "log.offset": 0, "service.type": "zeek", "tags": [ - "zeek" + "zeek.files" ], "zeek.files.analyzers": [ "X509", @@ -48,7 +48,7 @@ "log.offset": 452, "service.type": "zeek", "tags": [ - "zeek" + "zeek.files" ], "zeek.files.analyzers": [ "X509", diff --git a/x-pack/filebeat/module/zeek/http/manifest.yml b/x-pack/filebeat/module/zeek/http/manifest.yml index 6ee2cadec4c9..e98068206ee9 100644 --- a/x-pack/filebeat/module/zeek/http/manifest.yml +++ b/x-pack/filebeat/module/zeek/http/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/http.log - name: tags - default: [zeek] + default: [zeek.http] ingest_pipeline: ingest/pipeline.json input: config/http.yml diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json index 9d99db4f00f9..d47d043d9af6 100644 --- a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json @@ -25,7 +25,7 @@ "source.ip": "10.178.98.102", "source.port": 62995, "tags": [ - "zeek" + "zeek.http" ], "url.domain": "ocsp.apple.com", "url.original": "/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=", diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml new file mode 100644 index 000000000000..9cffd381595a --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/config/notice.yml @@ -0,0 +1,113 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +tags: {{.tags}} + +json.keys_under_root: false + +processors: + - drop_fields: + fields: ["json.actions"] + - rename: + fields: + - from: "json" + to: "zeek.notice" + + - from: "zeek.notice.src" + to: "source.address" + when: + has_fields: ["zeek.notice.src"] + + - from: "zeek.notice.dest" + to: "destination.address" + when: + has_fields: ["zeek.notice.dest"] + + - from: "zeek.notice.uid" + to: "zeek.session_id" + when: + has_fields: ["zeek.notice.uid"] + + - from: "zeek.notice.p" + to: "destination.port" + when: + has_fields: ["zeek.notice.p"] + + - from: "zeek.notice.conn" + to: "zeek.notice.connnection_id" + when: + has_fields: ["zeek.notice.conn"] + + - from: "zeek.notice.iconn" + to: "zeek.notice.icmp_id" + when: + has_fields: ["zeek.notice.iconn"] + + - from: "zeek.notice.id.orig_h" + to: "source.address" + when: + has_fields: ["zeek.notice.id.orig_h"] + + - from: "zeek.notice.id.orig_p" + to: "source.port" + when: + has_fields: ["zeek.notice.id.orig_p"] + + - from: "zeek.notice.id.resp_h" + to: "destination.address" + when: + has_fields: ["zeek.notice.id.resp_h"] + + - from: "zeek.notice.id.resp_p" + to: "destination.port" + when: + has_fields: ["zeek.notice.id.resp_p"] + + - from: "zeek.notice.proto" + to: "network.transport" + when: + has_fields: ["zeek.notice.proto"] + + - from: "zeek.notice.id.orig_p" + to: "source.port" + when: + has_fields: ["zeek.notice.id.orig_p"] + + - from: "zeek.notice.f.id" + to: "zeek.notice.file.id" + when: + has_fields: ["zeek.notice.f.id"] + + - from: "zeek.notice.f.parent_id" + to: "dzeek.notice.file.parent_id" + when: + has_fields: ["zeek.notice.f.parent_id"] + + - from: "zeek.notice.f.source" + to: "zeek.notice.file.source" + when: + has_fields: ["zeek.notice.f.source"] + + - from: "zeek.notice.f.is_orig" + to: "zeek.notice.file.is_orig" + when: + has_fields: ["zeek.notice.f.is_orig"] + + - from: "zeek.notice.f.seen_bytes" + to: "zeek.notice.file.seen_bytes" + when: + has_fields: ["zeek.notice.f.seen_bytes"] + + - from: "zeek.notice.f.total_bytes" + to: "zeek.notice.file.total_bytes" + when: + has_fields: ["zeek.notice.f.total_bytes"] + + ignore_missing: true + fail_on_error: false + + - drop_fields: + fields: ["zeek.notice.remote_location", "zeek.notice.f"] diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json new file mode 100644 index 000000000000..e41e1a0ad921 --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json @@ -0,0 +1,47 @@ +{ + "description": "Pipeline for normalizing Zeek notice.log", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * params.multiplier; ctx.zeek.notice.remove('ts');", + "params": { + "multiplier": 1000 + } + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx['destination'] != null && ctx['destination']['address'] != null) { ctx.destination.ip = ctx['destination']['address']; }" + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx['source'] != null && ctx['source']['address'] != null) { ctx.source.ip = ctx['source']['address']; }" + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.id = ctx.zeek.session_id + \"-notice\";", + "ignore_failure": true + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + } + ] +} diff --git a/x-pack/filebeat/module/zeek/notice/manifest.yml b/x-pack/filebeat/module/zeek/notice/manifest.yml new file mode 100644 index 000000000000..b806ac04e1d7 --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/bro/current/notice.log + os.linux: + - /var/log/bro/current/notice.log + os.darwin: + - /usr/local/var/logs/current/notice.log + - name: tags + default: [zeek.notice] + +ingest_pipeline: ingest/pipeline.json +input: config/notice.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log b/x-pack/filebeat/module/zeek/notice/test/notice-json.log new file mode 100644 index 000000000000..8c20486cb79f --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/test/notice-json.log @@ -0,0 +1 @@ +{"ts":1320435875.879278,"note":"SSH::Password_Guessing","msg":"172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).","sub":"Sampled servers: 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136","src":"172.16.238.1","peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false} diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json new file mode 100644 index 000000000000..26bfbcb4e348 --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json @@ -0,0 +1,24 @@ +[ + { + "@timestamp": 1320435875000, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.notice", + "event.id": "null-notice", + "event.module": "zeek", + "fileset.name": "notice", + "input.type": "log", + "log.offset": 0, + "service.type": "zeek", + "source.address": "172.16.238.1", + "source.ip": "172.16.238.1", + "tags": [ + "zeek.notice" + ], + "zeek.notice.dropped": false, + "zeek.notice.msg": "172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).", + "zeek.notice.note": "SSH::Password_Guessing", + "zeek.notice.peer_descr": "bro", + "zeek.notice.sub": "Sampled servers: 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136", + "zeek.notice.suppress_for": 3600 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/ssl/manifest.yml b/x-pack/filebeat/module/zeek/ssl/manifest.yml index d403fa973112..74d9c46134f2 100644 --- a/x-pack/filebeat/module/zeek/ssl/manifest.yml +++ b/x-pack/filebeat/module/zeek/ssl/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/ssl.log - name: tags - default: [zeek] + default: [zeek.ssl] ingest_pipeline: ingest/pipeline.json input: config/ssl.yml diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json index 3ef9fd2bb8d7..e61c0b33f645 100644 --- a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json @@ -23,7 +23,7 @@ "source.ip": "10.178.98.102", "source.port": 63199, "tags": [ - "zeek" + "zeek.ssl" ], "zeek.session_id": "CAOvs1BMFCX2Eh0Y3", "zeek.ssl.cert_chain_fuids": [ @@ -66,7 +66,7 @@ "source.ip": "10.178.98.102", "source.port": 63198, "tags": [ - "zeek" + "zeek.ssl" ], "zeek.session_id": "C3mki91FnnNtm0u1ok", "zeek.ssl.cert_chain_fuids": [ diff --git a/x-pack/filebeat/modules.d/zeek.yml.disabled b/x-pack/filebeat/modules.d/zeek.yml.disabled index 6cf23b0b8238..c43668021eab 100644 --- a/x-pack/filebeat/modules.d/zeek.yml.disabled +++ b/x-pack/filebeat/modules.d/zeek.yml.disabled @@ -13,6 +13,8 @@ enabled: true ssl: enabled: true + notice: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. From 0c89f5456598b2db9e2921262a0440a7f1b15691 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 25 Feb 2019 16:01:41 -0800 Subject: [PATCH 06/16] Update field descriptions, ingest processors, and others --- filebeat/docs/fields.asciidoc | 418 +++++++++++++++++- x-pack/filebeat/module/zeek/_meta/fields.yml | 294 +++++++++++- .../zeek/connection/config/connection.yml | 2 +- .../zeek/connection/ingest/pipeline.json | 21 +- .../test/connection-json.log-expected.json | 2 +- .../module/zeek/dns/ingest/pipeline.json | 9 +- .../zeek/dns/test/dns-json.log-expected.json | 2 +- x-pack/filebeat/module/zeek/fields.go | 2 +- .../module/zeek/files/ingest/pipeline.json | 10 +- .../files/test/files-json.log-expected.json | 4 +- .../module/zeek/http/ingest/pipeline.json | 9 +- .../http/test/http-json.log-expected.json | 2 +- .../module/zeek/notice/config/notice.yml | 39 +- .../module/zeek/notice/ingest/pipeline.json | 15 +- .../notice/test/notice-json.log-expected.json | 1 - .../module/zeek/ssl/ingest/pipeline.json | 9 +- .../zeek/ssl/test/ssl-json.log-expected.json | 4 +- 17 files changed, 729 insertions(+), 114 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index c454f1766eef..b55a7b9775d6 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -13480,6 +13480,9 @@ Fields from Zeek/Bro logs after normalization -- type: keyword +A unique identifier of the session + + -- *`zeek.connection.local_orig`*:: @@ -13487,6 +13490,9 @@ type: keyword -- type: boolean +Indicates whether the session is originated locally + + -- *`zeek.connection.local_resp`*:: @@ -13494,6 +13500,9 @@ type: boolean -- type: boolean +Indicates whether the session is responded locally + + -- *`zeek.connection.missed_bytes`*:: @@ -13501,6 +13510,9 @@ type: boolean -- type: long +Missed bytes for the session + + -- *`zeek.connection.state`*:: @@ -13508,6 +13520,9 @@ type: long -- type: keyword +Flags indicating the state of the session + + -- *`zeek.connection.history`*:: @@ -13515,6 +13530,9 @@ type: keyword -- type: keyword +Flags indicating the history of the session + + -- *`zeek.connection.orig_l2_addr`*:: @@ -13522,33 +13540,48 @@ type: keyword -- type: keyword +Link-layer address of the originator, if available + + -- -*`zeek.resp_l2_addr`*:: +*`zeek.connection.resp_l2_addr`*:: + -- type: keyword +Link-layer address of the responder, if available + + -- -*`zeek.vlan`*:: +*`zeek.connection.vlan`*:: + -- -type: keyword +type: integer + +VLAN identifier + -- -*`zeek.inner_vlan`*:: +*`zeek.connection.inner_vlan`*:: + -- -type: keyword +type: integer + +VLAN identifier + -- *`zeek.dns.trans_id`*:: + -- -type: integer +type: keyword + +DNS transaction identifier + -- @@ -13557,6 +13590,9 @@ type: integer -- type: double +Round trip time for the query and response + + -- *`zeek.dns.query`*:: @@ -13564,6 +13600,9 @@ type: double -- type: keyword +The domain name that is the subject of the DNS query + + -- *`zeek.dns.qclass`*:: @@ -13571,6 +13610,9 @@ type: keyword -- type: long +The QCLASS value specifying the class of the query + + -- *`zeek.dns.qclass_name`*:: @@ -13578,6 +13620,9 @@ type: long -- type: keyword +A descriptive name for the class of the query + + -- *`zeek.dns.qtype`*:: @@ -13585,6 +13630,9 @@ type: keyword -- type: long +A QTYPE value specifying the type of the query + + -- *`zeek.dns.qtype_name`*:: @@ -13592,6 +13640,9 @@ type: long -- type: keyword +A descriptive name for the type of the query + + -- *`zeek.dns.rcode`*:: @@ -13599,6 +13650,9 @@ type: keyword -- type: long +The response code value in DNS response messages + + -- *`zeek.dns.rcode_name`*:: @@ -13606,6 +13660,9 @@ type: long -- type: keyword +A descriptive name for the response code value + + -- *`zeek.dns.AA`*:: @@ -13613,6 +13670,10 @@ type: keyword -- type: boolean +The Authoritative Answer bit for response messages specifies that the responding +name server is an authority for the domain name in the question section + + -- *`zeek.dns.TC`*:: @@ -13620,6 +13681,9 @@ type: boolean -- type: boolean +The Truncation bit specifies that the message was truncated + + -- *`zeek.dns.RD`*:: @@ -13627,6 +13691,10 @@ type: boolean -- type: boolean +The Recursion Desired bit in a request message indicates that the client +wants recursive service for this query + + -- *`zeek.dns.RA`*:: @@ -13634,6 +13702,10 @@ type: boolean -- type: boolean +The Recursion Available bit in a response message indicates that the name +server supports recursive queries. + + -- *`zeek.dns.answers`*:: @@ -13641,6 +13713,9 @@ type: boolean -- type: keyword +The set of resource descriptions in the query answer + + -- *`zeek.dns.TTLs`*:: @@ -13648,6 +13723,9 @@ type: keyword -- type: double +The caching intervals of the associated RRs described by the answers field + + -- *`zeek.dns.rejected`*:: @@ -13655,6 +13733,9 @@ type: double -- type: boolean +Indicates whether the DNS query was rejected by the server + + -- *`zeek.dns.total_answers`*:: @@ -13662,6 +13743,9 @@ type: boolean -- type: integer +The total number of resource records in the reply + + -- *`zeek.dns.total_replies`*:: @@ -13669,6 +13753,9 @@ type: integer -- type: integer +The total number of resource records in the reply message + + -- *`zeek.dns.saw_query`*:: @@ -13676,6 +13763,9 @@ type: integer -- type: boolean +Whether the full DNS query has been seen + + -- *`zeek.dns.saw_reply`*:: @@ -13683,6 +13773,9 @@ type: boolean -- type: boolean +Whether the full DNS reply has been seen + + -- *`zeek.http.trans_depth`*:: @@ -13690,6 +13783,9 @@ type: boolean -- type: integer +Represents the pipelined depth into the connection of this request/response transaction + + -- *`zeek.http.status_msg`*:: @@ -13697,6 +13793,9 @@ type: integer -- type: keyword +Status message returned by the server + + -- *`zeek.http.info_code`*:: @@ -13704,6 +13803,9 @@ type: keyword -- type: integer +Last seen 1xx informational reply code returned by the server. + + -- *`zeek.http.info_msg`*:: @@ -13711,20 +13813,30 @@ type: integer -- type: keyword +Last seen 1xx informational reply message returned by the server. + + -- -*`zeek.http.filename`*:: +*`zeek.http.tags`*:: + -- type: keyword +A set of indicators of various attributes discovered and related to a particular +request/response pair. + + -- -*`zeek.http.tags`*:: +*`zeek.http.password`*:: + -- type: keyword +Password if basic-auth is performed for the request + + -- *`zeek.http.captured_password`*:: @@ -13732,6 +13844,9 @@ type: keyword -- type: boolean +Determines if the password will be captured for this request + + -- *`zeek.http.proxied`*:: @@ -13739,6 +13854,9 @@ type: boolean -- type: keyword +All of the headers that may indicate if the HTTP request was proxied + + -- *`zeek.http.range_request`*:: @@ -13746,6 +13864,9 @@ type: keyword -- type: boolean +Indicates if this request can assume 206 partial content in response + + -- *`zeek.http.client_header_names`*:: @@ -13753,6 +13874,10 @@ type: boolean -- type: keyword +The vector of HTTP header names sent by the client. No header values +are included here, just the header names. + + -- *`zeek.http.server_header_names`*:: @@ -13760,6 +13885,10 @@ type: keyword -- type: keyword +The vector of HTTP header names sent by the server. No header values +are included here, just the header names + + -- *`zeek.http.orig_fuids`*:: @@ -13767,6 +13896,9 @@ type: keyword -- type: keyword +An ordered vector of file unique IDs from the originator + + -- *`zeek.http.orig_mime_types`*:: @@ -13774,6 +13906,9 @@ type: keyword -- type: keyword +An ordered vector of mime types from the originator + + -- *`zeek.http.orig_filenames`*:: @@ -13781,6 +13916,9 @@ type: keyword -- type: keyword +An ordered vector of filenames from the originator + + -- *`zeek.http.resp_fuids`*:: @@ -13788,6 +13926,9 @@ type: keyword -- type: keyword +An ordered vector of file unique IDs from the responder + + -- *`zeek.http.resp_mime_types`*:: @@ -13795,6 +13936,9 @@ type: keyword -- type: keyword +An ordered vector of mime types from the responder + + -- *`zeek.http.resp_filenames`*:: @@ -13802,6 +13946,9 @@ type: keyword -- type: keyword +An ordered vector of filenames from the responder + + -- *`zeek.http.orig_mime_depth`*:: @@ -13809,6 +13956,9 @@ type: keyword -- type: integer +Current number of MIME entities in the HTTP request message body + + -- *`zeek.http.resp_mime_depth`*:: @@ -13816,6 +13966,9 @@ type: integer -- type: integer +Current number of MIME entities in the HTTP response message body + + -- *`zeek.files.fuid`*:: @@ -13823,6 +13976,9 @@ type: integer -- type: keyword +A file unique identifier + + -- *`zeek.files.tx_host`*:: @@ -13830,6 +13986,9 @@ type: keyword -- type: ip +The host that transferred the file + + -- *`zeek.files.rx_host`*:: @@ -13837,6 +13996,9 @@ type: ip -- type: ip +The host that received the file + + -- *`zeek.files.session_ids`*:: @@ -13844,6 +14006,9 @@ type: ip -- type: keyword +The sessions that have this file + + -- *`zeek.files.source`*:: @@ -13851,6 +14016,11 @@ type: keyword -- type: keyword +An identification of the source of the file data. E.g. it may be a network protocol +over which it was transferred, or a local file path which was read, or some other +input source + + -- *`zeek.files.depth`*:: @@ -13858,6 +14028,11 @@ type: keyword -- type: long +A value to represent the depth of this file in relation to its source. In SMTP, it +is the depth of the MIME attachment on the message. In HTTP, it is the depth of the +request within the TCP connection + + -- *`zeek.files.analyzers`*:: @@ -13865,6 +14040,9 @@ type: long -- type: keyword +A set of analysis types done during the file analysis + + -- *`zeek.files.mime_type`*:: @@ -13872,6 +14050,9 @@ type: keyword -- type: keyword +Mime type of the file + + -- *`zeek.files.filename`*:: @@ -13879,6 +14060,9 @@ type: keyword -- type: keyword +Name of the file if available + + -- *`zeek.files.local_orig`*:: @@ -13886,6 +14070,10 @@ type: keyword -- type: boolean +If the source of this file is a network connection, this field indicates if the data +originated from the local network or not + + -- *`zeek.files.is_orig`*:: @@ -13893,6 +14081,10 @@ type: boolean -- type: boolean +If the source of this file is a network connection, this field indicates if the file is +being sent by the originator of the connection or the responder + + -- *`zeek.files.duration`*:: @@ -13900,6 +14092,9 @@ type: boolean -- type: double +The duration the file was analyzed for. Not the duration of the session. + + -- *`zeek.files.seen_bytes`*:: @@ -13907,6 +14102,9 @@ type: double -- type: long +Number of bytes provided to the file analysis engine for the file + + -- *`zeek.files.total_bytes`*:: @@ -13914,6 +14112,9 @@ type: long -- type: long +Total number of bytes that are supposed to comprise the full file + + -- *`zeek.files.missing_bytes`*:: @@ -13921,6 +14122,10 @@ type: long -- type: long +The number of bytes in the file stream that were completely missed during the process +of analysis + + -- *`zeek.files.overflow_bytes`*:: @@ -13928,6 +14133,10 @@ type: long -- type: long +The number of bytes in the file stream that were not delivered to stream file analyzers. +This could be overlapping bytes or bytes that couldn't be reassembled + + -- *`zeek.files.timedout`*:: @@ -13935,6 +14144,9 @@ type: long -- type: boolean +Whether the file analysis timed out at least once for the file + + -- *`zeek.files.parent_fuid`*:: @@ -13942,6 +14154,10 @@ type: boolean -- type: keyword +Identifier associated with a container file from which this one was extracted as part of +the file analysis + + -- *`zeek.files.md5`*:: @@ -13949,6 +14165,9 @@ type: keyword -- type: keyword +An MD5 digest of the file contents + + -- *`zeek.files.sha1`*:: @@ -13956,6 +14175,9 @@ type: keyword -- type: keyword +A SHA1 digest of the file contents + + -- *`zeek.files.sha256`*:: @@ -13963,6 +14185,9 @@ type: keyword -- type: keyword +A SHA256 digest of the file contents. + + -- *`zeek.files.extracted`*:: @@ -13970,6 +14195,9 @@ type: keyword -- type: keyword +Local filename of extracted file + + -- *`zeek.files.extracted_cutoff`*:: @@ -13977,6 +14205,9 @@ type: keyword -- type: boolean +Indicate whether the file being extracted was cut off hence not extracted completely + + -- *`zeek.files.extracted_size`*:: @@ -13984,6 +14215,9 @@ type: boolean -- type: long +The number of bytes extracted to disk + + -- *`zeek.files.entropy`*:: @@ -13991,6 +14225,9 @@ type: long -- type: double +The information density of the contents of the file + + -- *`zeek.ssl.version`*:: @@ -13998,6 +14235,9 @@ type: double -- type: keyword +SSL/TLS version that was logged + + -- *`zeek.ssl.cipher`*:: @@ -14005,6 +14245,9 @@ type: keyword -- type: keyword +SSL/TLS cipher suite that was logged + + -- *`zeek.ssl.curve`*:: @@ -14012,6 +14255,9 @@ type: keyword -- type: keyword +Elliptic curve that was logged when using ECDH/ECDHE + + -- *`zeek.ssl.server_name`*:: @@ -14019,6 +14265,10 @@ type: keyword -- type: keyword +Value of the Server Name Indicator SSL/TLS extension. It indicates the server name +that the client was requesting + + -- *`zeek.ssl.resumed`*:: @@ -14026,6 +14276,10 @@ type: keyword -- type: boolean +Flag to indicate if the session was resumed reusing the key material exchanged in an +earlier connection + + -- *`zeek.ssl.next_protocol`*:: @@ -14033,6 +14287,9 @@ type: boolean -- type: keyword +Next protocol the server chose using the application layer next protocol extension + + -- *`zeek.ssl.established`*:: @@ -14040,6 +14297,9 @@ type: keyword -- type: boolean +Flag to indicate if this ssl session has been established successfully + + -- *`zeek.ssl.cert_chain`*:: @@ -14047,6 +14307,9 @@ type: boolean -- type: keyword +Chain of certificates offered by the server to validate its complete signing chain + + -- *`zeek.ssl.cert_chain_fuids`*:: @@ -14054,6 +14317,9 @@ type: keyword -- type: keyword +An ordered vector of certificate file identifiers for the certificates offered by the server + + -- *`zeek.ssl.client_cert_chain`*:: @@ -14061,6 +14327,9 @@ type: keyword -- type: keyword +Chain of certificates offered by the client to validate its complete signing chain + + -- *`zeek.ssl.client_cert_chain_fuids`*:: @@ -14068,6 +14337,9 @@ type: keyword -- type: keyword +An ordered vector of certificate file identifiers for the certificates offered by the client + + -- *`zeek.ssl.issuer`*:: @@ -14075,6 +14347,9 @@ type: keyword -- type: keyword +Subject of the signer of the X.509 certificate offered by the server + + -- *`zeek.ssl.client_issuer`*:: @@ -14082,6 +14357,9 @@ type: keyword -- type: keyword +Subject of the X.509 certificate offered by the client + + -- *`zeek.ssl.validation_status`*:: @@ -14089,6 +14367,19 @@ type: keyword -- type: keyword +Result of certificate validation for this connection + + +-- + +*`zeek.ssl.validation_code`*:: ++ +-- +type: keyword + +Result of certificate validation for this connection, given as OpenSSL validation code + + -- *`zeek.ssl.subject`*:: @@ -14096,6 +14387,9 @@ type: keyword -- type: keyword +Subject of the X.509 certificate offered by the server + + -- *`zeek.ssl.client_subject`*:: @@ -14103,6 +14397,9 @@ type: keyword -- type: keyword +Subject of the X.509 certificate offered by the client + + -- *`zeek.ssl.last_alert`*:: @@ -14110,6 +14407,9 @@ type: keyword -- type: keyword +Last alert that was seen during the connection + + -- *`zeek.notice.connection_id`*:: @@ -14117,6 +14417,9 @@ type: keyword -- type: keyword +Identifier of the related connection session + + -- *`zeek.notice.icmp_id`*:: @@ -14124,6 +14427,9 @@ type: keyword -- type: keyword +Identifier of the related ICMP session + + -- *`zeek.notice.file.id`*:: @@ -14131,6 +14437,9 @@ type: keyword -- type: keyword +An identifier associated with a single file that is related to this notice + + -- *`zeek.notice.file.parent_id`*:: @@ -14138,6 +14447,9 @@ type: keyword -- type: keyword +Identifier associated with a container file from which this one was extracted + + -- *`zeek.notice.file.source`*:: @@ -14145,6 +14457,21 @@ type: keyword -- type: keyword +An identification of the source of the file data. E.g. it may be a network protocol +over which it was transferred, or a local file path which was read, or some other +input source + + +-- + +*`zeek.notice.file.mime_type`*:: ++ +-- +type: keyword + +A mime type if the notice is related to a file + + -- *`zeek.notice.file.is_orig`*:: @@ -14152,6 +14479,10 @@ type: keyword -- type: boolean +If the source of this file is a network connection, this field indicates if the file is +being sent by the originator of the connection or the responder + + -- *`zeek.notice.file.seen_bytes`*:: @@ -14159,27 +14490,51 @@ type: boolean -- type: long +Number of bytes provided to the file analysis engine for the file + + -- -*`zeek.notice.file.total_bytes`*:: +*`zeek.fnotice.file.total_bytes`*:: + -- type: long +Total number of bytes that are supposed to comprise the full file + + -- -*`zeek.notice.fuid`*:: +*`zeek.notice.file.missing_bytes`*:: + -- -type: keyword +type: long + +The number of bytes in the file stream that were completely missed during the process +of analysis + -- -*`zeek.notice.file_mime_type`*:: +*`zeek.notice.file.overflow_bytes`*:: ++ +-- +type: long + +The number of bytes in the file stream that were not delivered to stream file analyzers. +This could be overlapping bytes or bytes that couldn't be reassembled + + +-- + +*`zeek.notice.fuid`*:: + -- type: keyword +A file unique ID if this notice is related to a file + + -- *`zeek.notice.note`*:: @@ -14187,6 +14542,9 @@ type: keyword -- type: keyword +The type of the notice + + -- *`zeek.notice.msg`*:: @@ -14194,6 +14552,9 @@ type: keyword -- type: keyword +The human readable message for the notice. + + -- *`zeek.notice.sub`*:: @@ -14201,6 +14562,9 @@ type: keyword -- type: keyword +The human readable sub-message + + -- *`zeek.notice.n`*:: @@ -14208,6 +14572,9 @@ type: keyword -- type: long +Associated count, or a status code + + -- *`zeek.notice.peer_name`*:: @@ -14215,6 +14582,9 @@ type: long -- type: keyword +Name of remote peer that raised this notice + + -- *`zeek.notice.peer_descr`*:: @@ -14222,6 +14592,9 @@ type: keyword -- type: text +Textual description for the peer that raised this notice + + -- *`zeek.notice.actions`*:: @@ -14229,6 +14602,9 @@ type: text -- type: keyword +The actions which have been applied to this notice + + -- *`zeek.notice.email_body_sections`*:: @@ -14236,6 +14612,10 @@ type: keyword -- type: text +By adding chunks of text into this element, other scripts can expand on notices +that are being emailed + + -- *`zeek.notice.email_delay_tokens`*:: @@ -14243,6 +14623,11 @@ type: text -- type: keyword +Adding a string token to this set will cause the built-in emailing functionality +to delay sending the email either the token has been removed or the email +has been delayed for the specified time duration + + -- *`zeek.notice.identifier`*:: @@ -14250,6 +14635,9 @@ type: keyword -- type: keyword +This field is provided when a notice is generated for the purpose of deduplicating notices + + -- *`zeek.notice.suppress_for`*:: @@ -14257,6 +14645,9 @@ type: keyword -- type: double +This field indicates the length of time that this unique notice should be suppressed + + -- *`zeek.notice.dropped`*:: @@ -14264,5 +14655,8 @@ type: double -- type: boolean +Indicate if the source IP address was dropped and denied network access + + -- diff --git a/x-pack/filebeat/module/zeek/_meta/fields.yml b/x-pack/filebeat/module/zeek/_meta/fields.yml index 912e86ef7dcb..cba71f9f4e7f 100644 --- a/x-pack/filebeat/module/zeek/_meta/fields.yml +++ b/x-pack/filebeat/module/zeek/_meta/fields.yml @@ -10,345 +10,609 @@ fields: - name: session_id type: keyword + description: > + A unique identifier of the session - name: connection.local_orig type: boolean + description: > + Indicates whether the session is originated locally - name: connection.local_resp type: boolean + description: > + Indicates whether the session is responded locally - name: connection.missed_bytes type: long + description: > + Missed bytes for the session - name: connection.state type: keyword + description: > + Flags indicating the state of the session - name: connection.history type: keyword + description: > + Flags indicating the history of the session - name: connection.orig_l2_addr type: keyword + description: > + Link-layer address of the originator, if available - - name: resp_l2_addr + - name: connection.resp_l2_addr type: keyword + description: > + Link-layer address of the responder, if available - - name: vlan - type: keyword + - name: connection.vlan + type: integer + description: > + VLAN identifier - - name: inner_vlan - type: keyword + - name: connection.inner_vlan + type: integer + description: > + VLAN identifier - name: dns.trans_id - type: integer + type: keyword + description: > + DNS transaction identifier - name: dns.rtt type: double + description: > + Round trip time for the query and response - name: dns.query type: keyword + description: > + The domain name that is the subject of the DNS query - name: dns.qclass type: long + description: > + The QCLASS value specifying the class of the query - name: dns.qclass_name type: keyword + description: > + A descriptive name for the class of the query - name: dns.qtype type: long + description: > + A QTYPE value specifying the type of the query - name: dns.qtype_name type: keyword + description: > + A descriptive name for the type of the query - name: dns.rcode type: long + description: > + The response code value in DNS response messages - name: dns.rcode_name type: keyword + description: > + A descriptive name for the response code value - name: dns.AA type: boolean + description: | + The Authoritative Answer bit for response messages specifies that the responding + name server is an authority for the domain name in the question section - name: dns.TC type: boolean + description: > + The Truncation bit specifies that the message was truncated - name: dns.RD type: boolean + description: | + The Recursion Desired bit in a request message indicates that the client + wants recursive service for this query - name: dns.RA type: boolean + description: | + The Recursion Available bit in a response message indicates that the name + server supports recursive queries. - name: dns.answers type: keyword + description: > + The set of resource descriptions in the query answer - name: dns.TTLs type: double + description: > + The caching intervals of the associated RRs described by the answers field - name: dns.rejected type: boolean + description: > + Indicates whether the DNS query was rejected by the server - name: dns.total_answers type: integer + description: > + The total number of resource records in the reply - name: dns.total_replies type: integer + description: > + The total number of resource records in the reply message - name: dns.saw_query type: boolean + description: > + Whether the full DNS query has been seen - name: dns.saw_reply type: boolean + description: > + Whether the full DNS reply has been seen - name: http.trans_depth type: integer + description: > + Represents the pipelined depth into the connection of this request/response transaction - name: http.status_msg type: keyword + description: > + Status message returned by the server - name: http.info_code type: integer + description: > + Last seen 1xx informational reply code returned by the server. - name: http.info_msg type: keyword + description: > + Last seen 1xx informational reply message returned by the server. - - name: http.filename + - name: http.tags type: keyword + description: | + A set of indicators of various attributes discovered and related to a particular + request/response pair. - - name: http.tags + + - name: http.password type: keyword + description: > + Password if basic-auth is performed for the request - name: http.captured_password type: boolean + description: > + Determines if the password will be captured for this request - name: http.proxied type: keyword + description: > + All of the headers that may indicate if the HTTP request was proxied - name: http.range_request type: boolean + description: > + Indicates if this request can assume 206 partial content in response - name: http.client_header_names type: keyword + description: | + The vector of HTTP header names sent by the client. No header values + are included here, just the header names. - name: http.server_header_names type: keyword + description: | + The vector of HTTP header names sent by the server. No header values + are included here, just the header names - name: http.orig_fuids type: keyword + description: > + An ordered vector of file unique IDs from the originator - name: http.orig_mime_types type: keyword + description: > + An ordered vector of mime types from the originator - name: http.orig_filenames type: keyword + description: > + An ordered vector of filenames from the originator - name: http.resp_fuids type: keyword + description: > + An ordered vector of file unique IDs from the responder - name: http.resp_mime_types type: keyword + description: > + An ordered vector of mime types from the responder - name: http.resp_filenames type: keyword + description: > + An ordered vector of filenames from the responder - name: http.orig_mime_depth type: integer + description: > + Current number of MIME entities in the HTTP request message body - name: http.resp_mime_depth type: integer + description: > + Current number of MIME entities in the HTTP response message body - name: files.fuid type: keyword + description: > + A file unique identifier - name: files.tx_host type: ip + description: > + The host that transferred the file - name: files.rx_host type: ip + description: > + The host that received the file - name: files.session_ids type: keyword + description: > + The sessions that have this file - name: files.source type: keyword + description: | + An identification of the source of the file data. E.g. it may be a network protocol + over which it was transferred, or a local file path which was read, or some other + input source - name: files.depth type: long - - - names: files.direction - type: keyword + description: | + A value to represent the depth of this file in relation to its source. In SMTP, it + is the depth of the MIME attachment on the message. In HTTP, it is the depth of the + request within the TCP connection - name: files.analyzers type: keyword + description: > + A set of analysis types done during the file analysis - name: files.mime_type type: keyword + description: > + Mime type of the file - name: files.filename type: keyword + description: > + Name of the file if available - name: files.local_orig type: boolean + description: | + If the source of this file is a network connection, this field indicates if the data + originated from the local network or not - name: files.is_orig type: boolean + description: | + If the source of this file is a network connection, this field indicates if the file is + being sent by the originator of the connection or the responder - name: files.duration type: double + description: > + The duration the file was analyzed for. Not the duration of the session. - name: files.seen_bytes type: long + description: > + Number of bytes provided to the file analysis engine for the file - name: files.total_bytes type: long + description: > + Total number of bytes that are supposed to comprise the full file - name: files.missing_bytes type: long + description: | + The number of bytes in the file stream that were completely missed during the process + of analysis - name: files.overflow_bytes type: long + description: | + The number of bytes in the file stream that were not delivered to stream file analyzers. + This could be overlapping bytes or bytes that couldn't be reassembled - name: files.timedout type: boolean + description: > + Whether the file analysis timed out at least once for the file - name: files.parent_fuid type: keyword + description: | + Identifier associated with a container file from which this one was extracted as part of + the file analysis - name: files.md5 type: keyword + description: > + An MD5 digest of the file contents - name: files.sha1 type: keyword + description: > + A SHA1 digest of the file contents - name: files.sha256 type: keyword + description: > + A SHA256 digest of the file contents. - name: files.extracted type: keyword + description: > + Local filename of extracted file - name: files.extracted_cutoff type: boolean + description: > + Indicate whether the file being extracted was cut off hence not extracted completely - name: files.extracted_size type: long + description: > + The number of bytes extracted to disk - name: files.entropy type: double + description: > + The information density of the contents of the file - name: ssl.version type: keyword + description: > + SSL/TLS version that was logged - name: ssl.cipher type: keyword + description: > + SSL/TLS cipher suite that was logged - name: ssl.curve type: keyword + description: > + Elliptic curve that was logged when using ECDH/ECDHE - name: ssl.server_name type: keyword + description: | + Value of the Server Name Indicator SSL/TLS extension. It indicates the server name + that the client was requesting - name: ssl.resumed type: boolean + description: | + Flag to indicate if the session was resumed reusing the key material exchanged in an + earlier connection - name: ssl.next_protocol type: keyword + description: > + Next protocol the server chose using the application layer next protocol extension - name: ssl.established type: boolean + description: > + Flag to indicate if this ssl session has been established successfully - name: ssl.cert_chain type: keyword + description: > + Chain of certificates offered by the server to validate its complete signing chain - name: ssl.cert_chain_fuids type: keyword + description: > + An ordered vector of certificate file identifiers for the certificates offered by the server - name: ssl.client_cert_chain type: keyword + description: > + Chain of certificates offered by the client to validate its complete signing chain - name: ssl.client_cert_chain_fuids type: keyword + description: > + An ordered vector of certificate file identifiers for the certificates offered by the client - name: ssl.issuer type: keyword + description: > + Subject of the signer of the X.509 certificate offered by the server - name: ssl.client_issuer type: keyword + description: > + Subject of the X.509 certificate offered by the client - name: ssl.validation_status type: keyword + description: > + Result of certificate validation for this connection + + - name: ssl.validation_code + type: keyword + description: > + Result of certificate validation for this connection, given as OpenSSL validation code - name: ssl.subject type: keyword + description: > + Subject of the X.509 certificate offered by the server - name: ssl.client_subject type: keyword + description: > + Subject of the X.509 certificate offered by the client - name: ssl.last_alert type: keyword + description: > + Last alert that was seen during the connection - name: notice.connection_id type: keyword + description: > + Identifier of the related connection session - name: notice.icmp_id type: keyword + description: > + Identifier of the related ICMP session - name: notice.file.id type: keyword + description: > + An identifier associated with a single file that is related to this notice - name: notice.file.parent_id type: keyword + description: > + Identifier associated with a container file from which this one was extracted - name: notice.file.source type: keyword + description: | + An identification of the source of the file data. E.g. it may be a network protocol + over which it was transferred, or a local file path which was read, or some other + input source + + - name: notice.file.mime_type + type: keyword + description: > + A mime type if the notice is related to a file - name: notice.file.is_orig type: boolean + description: | + If the source of this file is a network connection, this field indicates if the file is + being sent by the originator of the connection or the responder - name: notice.file.seen_bytes type: long + description: > + Number of bytes provided to the file analysis engine for the file - - name: notice.file.total_bytes + - name: fnotice.file.total_bytes type: long + description: > + Total number of bytes that are supposed to comprise the full file - - name: notice.fuid - type: keyword + - name: notice.file.missing_bytes + type: long + description: | + The number of bytes in the file stream that were completely missed during the process + of analysis - - name: notice.file_mime_type + - name: notice.file.overflow_bytes + type: long + description: | + The number of bytes in the file stream that were not delivered to stream file analyzers. + This could be overlapping bytes or bytes that couldn't be reassembled + + - name: notice.fuid type: keyword + description: > + A file unique ID if this notice is related to a file - name: notice.note type: keyword + description: > + The type of the notice - name: notice.msg type: keyword + description: > + The human readable message for the notice. - name: notice.sub type: keyword + description: > + The human readable sub-message - name: notice.n type: long + description: > + Associated count, or a status code - name: notice.peer_name type: keyword + description: > + Name of remote peer that raised this notice - name: notice.peer_descr type: text + description: > + Textual description for the peer that raised this notice - name: notice.actions type: keyword + description: > + The actions which have been applied to this notice - name: notice.email_body_sections type: text + description: | + By adding chunks of text into this element, other scripts can expand on notices + that are being emailed - name: notice.email_delay_tokens type: keyword + description: | + Adding a string token to this set will cause the built-in emailing functionality + to delay sending the email either the token has been removed or the email + has been delayed for the specified time duration - name: notice.identifier type: keyword + description: > + This field is provided when a notice is generated for the purpose of deduplicating notices - name: notice.suppress_for type: double + description: > + This field indicates the length of time that this unique notice should be suppressed - name: notice.dropped type: boolean + description: > + Indicate if the source IP address was dropped and denied network access diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml index b925dc01aec0..47fb29066426 100644 --- a/x-pack/filebeat/module/zeek/connection/config/connection.yml +++ b/x-pack/filebeat/module/zeek/connection/config/connection.yml @@ -17,7 +17,7 @@ processors: to: "zeek.connection" - from: "zeek.connection.duration" - to: "event.duration" + to: "temp.duration" - from: "zeek.connection.id.orig_h" to: "source.address" diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json index 862787cd0f77..3393f112971a 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json @@ -4,27 +4,30 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['connection']['ts'] * params.multiplier; ctx.zeek.connection.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['connection']['ts'] * 1000; ctx.zeek.connection.remove('ts');" } }, { "script": { "lang": "painless", - "source": "ctx.event.duration = (long)ctx.event.duration * params.multiplier", + "source": "ctx.event.duration = (long)ctx.temp.duration * params.scale", "params": { - "multiplier": 1000000000 + "scale": 1000000000 }, - "ignore_failure": true + "if": "ctx.temp?.duration != null" + } + }, + { + "remove": { + "field": "temp.duration", + "ignore_missing": true } }, { "script": { "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-connection\"", - "ignore_failure": true + "source": "ctx.event.id = ctx['zeek']['session_id'];", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json index dc745da83a12..ce845d36c809 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json @@ -9,7 +9,7 @@ "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.connection", "event.duration": 0.0, - "event.id": "CAcJw21BbVedgFnYH3-connection", + "event.id": "CAcJw21BbVedgFnYH3", "event.module": "zeek", "fileset.name": "connection", "input.type": "log", diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json index 28f4adb5f41d..b4780fae27aa 100644 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json @@ -4,17 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['dns']['ts'] * params.multiplier; ctx.zeek.dns.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['dns']['ts'] * 1000; ctx.zeek.dns.remove('ts');" } }, { "script": { "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-dns\"", - "ignore_failure": true + "source": "ctx.event.id = ctx['zeek']['session_id'];", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json index 4a273973b425..acc66d7e0447 100644 --- a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json @@ -6,7 +6,7 @@ "destination.port": 53, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.dns", - "event.id": "CAcJw21BbVedgFnYH3-dns", + "event.id": "CAcJw21BbVedgFnYH3", "event.module": "zeek", "fileset.name": "dns", "input.type": "log", diff --git a/x-pack/filebeat/module/zeek/fields.go b/x-pack/filebeat/module/zeek/fields.go index bc440ae8bff3..4ecc23bb6dca 100644 --- a/x-pack/filebeat/module/zeek/fields.go +++ b/x-pack/filebeat/module/zeek/fields.go @@ -19,5 +19,5 @@ func init() { // AssetZeek returns asset data. // This is the base64 encoded gzipped contents of module/zeek. func AssetZeek() string { - return "eJycmMFy4zYMhu9+Cr7AutOd2R5y6My2nZ7aSyenXji0CElsKIILUE6Up++Q0tpxJEfgHm39HwGCAAHpk3qC6UG9AjwdlEoueXhQ/86/LHBDLiaH4UH9elBKqb/Rjh5Ui6R6E6x3oVMeO1aR0I4NWHWaCv7Tb4QHpVoH3vLDQalPKpgBLpaUSlOEB9URjrH83rCm1J+FVy3hcFl2NmjaBKQC0mC8ezUZK8zV4tUmA7PDoJ1dlp1tP8H0jGQPN9oGQ4AmL3f02BivkVx3g50QPZiwgxFwrMIGxwxWn6YEfAN6DN1dipNJULWr3nFCmqqYHAPtP2tjLe2Deety9dmbsK9yIQBpmdYGPiYygd8fuAsJOqC1mlK6EVocTx7Wum8jSCJXpI03vHeQV6HOfwhXzs8lC+eHFetSg1aybtFVrPv1634dZN3j7zLdP38IdUK7JvAzEMs28/j4F4tSheA/aBLYTR/yP+uUxWS83vLmbt7OCEH0DoQIm2e9TuO7wcnyvL5A3qcUl7qzEFO/6c9q54XKt9jIeuBu8xS2IRda1Kuc/dhQYe6ZWctb50GW6PPmTSdIoyJtTEwjgdXRMF82uR/fSPjiQNDHippM6EATfBuBk9BC4x2EpHswFqiUuXRPDHQG+hGy9Jd2dLYKGNwAOivrzCxnKoVKN6vxrQDVvs1mKn27huF+xX3koJDKbvExx2Dfr1mbXnSP71LOxS0liZXXKU4QngXBkRpQUvk6HO87IF+kjubZSLq2CcZPr6I+MwOXDJIC8stq1teMtzPhuEpuRzKrEG31y+/nC0E0/i4pVnqfXJ/naxe6CgLPQK3H5xqn3AAWR8FdO+ujoXzZ1pTWYL+I0783P1doP3/5RaqGl0RmNd5IAN2MCdtWGqArx+51bzpdkJAI47Sbdcz+eAZiURFnceNiD4I3mqId6SyoxCxd2qasdDNAwONwZ7JciQO8JB0JEzboZesDJ3PyjnupjQYo6aY3ThrIi17aWAs1DybVxt5jNTYd8yg+8tlQDXI23tlyR+p5BBZmzHjKLxdVblUx3nDSxgMJ9AGTa+B4/Vog+syyUK4ZYo0+V/ixVr9csrXYPDxUuibtkDeGpI3vLSRvf98pUY95Y0JXTCALFlDyMWoRi17DFi2PJ7kTsnhEEF+6b4nymfIGSfCSNvWmlIOgphc9DMZ5fUI7aYYN9q6hGbTgzaQTPkGFTWchJNc6yY11OYsYCZh1i7TbZBfGEsb4UTs5HA7/BwAA//9bGsLO" + return "eJzkW99vG7eyfvdfMW/3JVHaAClw83ABN05RA3Zuahu995wXgVqOJNZccktyZSvoH38wQ3J/SCtZu3GMnNOXALF25/uGHM58HHJfwz1u38MXxPszgKCCxvfwz/g/ib5wqgrKmvfwP2cAANdW1hphaR2shZFamRVou/JQOSvrAiUstvz6m5+dPQNYKtTSvz8DeA1GlNggAYRthe9h5Wxd8f8H0AB+4fdh6WzZmI2AYhnQgbGuFFp9EfQav9MitpgevVfWzJVMZiP2PW4frMt/G8QHOIfaqD9rBCXRBLVU6MAuIawxm4WzHlhhjcGCzMy0LYSeW6dWPdyFtRqFOY57aaQqREAPD2sMa3Q9SOWBzCojAkpgHL19godDX30DHmTWGnkCjVJ5j3K+2Ab0PSLamtVxFtf8KvCrHH0dEgfxfBABx8/4L1qsPKjoNwU4Y5GtnYk/CLxWPli3fSboZO1UcIqLuX47F1K68QyulLl/rcUWHZAB9D7j5niz7hWoJYiNUFosNB4kQoHxLYjkgDuVx0Y3MR7xlQm4Qncc//er80+dNX/QujIG3fwbYEjjZ8EJ4yelrYtPt8BvC6Z5FMWF0AOQtqbxPGr/xtZGQnCqgqBKbNbknzW6LQgj0zR53MfjZ8a7dLdGkLYUyrAtCGsRKAHxoqgXf2ARcoiQ9xFlH7zQwo9NPwT924er89tb2AhdI/gKC7Xc5iXKNjP4UeA5/WFKFWp+2GB0P4/4SdgENdLnc/jt7h+fPw47TEZOwXx+d09AdoWVY729axKLR6D3k9/KcDA1P5XovVihP4D6/P4OkNrHPj8/tbD/1fP4vA5r61QQjHtu/AM6WKjA6Hs+pyBQ6OPa6yRjiovGMvP36DboaH0KAyLhbBu3ugtZmTybnnOVj6l13827D+P1C7l552pTsERk5wbcSC7Cg/AQ4tMo9wncXEwb5xssaseC6QK9cqRkVCC/BThkvxsGqtFbDblCKzShHd8HYQLpLra5iUOtihwyyh9aFzcTo6Rlf56LbZd/P0qGHOBZbkymyPB1VVnXc4R4K/SzfeqCQ9NPKxoeuTA49LZ2BXYf9Z3g47pFMAOBd3flR5dIgi5Esaa1QXLAbYRuErXw3haK1fvNjU8mFnH/xL9Hh+N2ZiDZIJU7lM+l55uCySsgW89k4oyxiX2JYoPQ86H5OUkC0SCxCTB1uYh7q2aiHBbWyWaOHFZ6CwMqiSnQrwpfhEIK9X0mXjzM99XNSfPyf53ZWNZad6ZkLTwsECkx4kBWJFDm9Uyg0ccjoOsQqiRNJVZhPX7Ib7By6JHSGEFXqkKtDEpge2TDxszX6Oy4bni7yfnyTZN3Oip3P0KZKu3daj8v/Wp8/rjld5vk5jDUzjy9MhhXmaWd72mRkwboSvjAIw8/Pj4CGXIlFzCh0/ywHBimMzvGZ9IoPE3n+ADNhgJIrE7O5391lFLK5qnMWMcpdSOcsrUHEYJTi5rSm1S+sBukYhu3I5qzbbAgoBIuqKLWwrVlaS+wKqGI+D7zSnjfYTpiHD+nN2njuhBeFa9JGZFIqtDRoKLsCD/mM4BfiCrUDuV8kMhJC/8CA7pSGfREhddgpvagtIYFla6I0sqKw4wqZx8VTumuaZ0r4hqFpIrHqqEU20ZIZIa/3t19buQSlamMus/HCbPCeeY7enTaGqn6WQcKErPe1yXC2x9+inEkNOWpQBJNmQPb3jhvrOPm0VHeKUxYAFStNlgEy3WKxyQaZCQPlFXz4ouAM/hk8zO8f/Bt0AtHgq3QtUQJa3T4Cv6ofehMSDQ7tIDj0n5Bd1IueQZ3BrzhttmyVnKCyDw3YJ3kVNP6slQac+/48iL1sPuNtEM0SlXinPCfiwsZZCOjaJAHo6b1hBGJs3oaCW4gvsiUNC3FQyxeZEaeZPECE3KMQxubE2Xfh9o5Ws+tzr6+vP4IaIIKtB1PIruX6LOwWFi5PTo7L8JpZ6+7T4qG1M8oaCfMUS9AD3VsI0J4nK/tTm1T1dMbHXopbcxJNy/RUUiw/le7PfSI5L4ayWGBavMETHtGN3mXzwaSgFiLDcbSfRAwbvAmSNC2m56aSvlkJlpM/+O5lCKIGXycrWagoqpZIAgwGB6suycFE2xhdVvBSLHCw1oVa3ohtqOaeXoF1oGI52zRfiXCOj0eN+4iPuRtiWB5b9eYVqaqQyI5NCD7C+hQ27Qrx2ObNFjaB8Q9Xezv8UYub9qYK4sjHUcsWFDBJzIzuDRwe333+RX53PL1u5Ywrk0RgijWJUFZ023fsSFaqWxo6P1dtQ8PKqzTEr/78Lmz4RwaIWGE3n6Z1Idq9i1swxM3Tv7SGgRZu9xY55HKzwxxaCrReA7XueZ0Q3QwfaW6MB7ikyj7C+DwCV2EGn9E3gbf5f6ya2LNd1ZZO6mv8iOoZadRmTYYtFg7K7E9X2+qY1x52a51YGwY8kr578Sl/GYDsECKtK6qbiVYnrhu18UdUwUpb9SuvX4xtkGaX27ZUh5LC433nqT3U07Jz/ZPwmfD5QTNpJsGnxodEC8bVM5ulIyNg70FCmhWyrSHNYcWVOxQTqFzt9OYjKS4xNFmh5voPrIrbFk55bHt5h2iUyrvlVmNINTfsO3SUZ3p88GhKCPDB3TItDQG1FuIF0C66a5ytkDfic9OhhxiTuVxqe3Dy1A3NoBErWIXKdj8RBsDVAxm0EFQHgpba0l1nshqUVXkbESzrjuB/KD5r0DPOhTeY7nQu/2MFECqRGnrCb2MXou3F7tsE2wdQATQKDyV0+LpYK4ECeb5GI3bSXDtLarOEQhVYRDcRhHKoItMOe1GccM5jkolZQd8DE7w+YTw3IKh6WwQTqui8t2kPdT1xTuQakXKoVvlUv9nEMqvxY9T9MLtr+c/TsB6++6niWhv3/10DG8wzTZTMR7zqlGxJomGdl4PhV7zxLyog10uxy+H3NrrnX6xn7Eythwo0oqaxmIJa6SFQemg/b3Na8eZevVlymWE3UTVAgcLUvn7QVQTnK22kypxp6sPEo1XYdsRBBwCh3Wj93q2QT4kHh8Jt7dXb+6ubiEZSAlYeNB2tdrNh4RUqGqNE+6UZaD4PvhaBTwBrXabCVL4o9b0xwL4/V0Yij8DNZVh+Pjh4tc39M/HffDUZB2jxttk+zvvzdKk3cZjd1bol/nwpBkSfAw059bM4DL0zvCbqxz9U/zd2wlx+xnvcJjVviMOfV2efmDdOvGLFiveL+4cB+QrqBGYjYPDOKL0+z1uoRQBnRIa8LFYC0PjrgwI07qBwmkqRoc2fkTd4GOY5636hD0RPoZ2p98Z0GJtPULLWFSVzi2FePPR9F5t5mifIvogFlr59ZQrAcMjrDxZboa5OQ/uQIGvC5JvJDa3AwsHXZgXa6Em5IQP9BqFLhmJnRbST8slS7HegQBR3witJFMPvsnM4NXK0OBGDkf4PWuXucM47b8azdPeXX7arQG+8fzomw9rWtGTh3WX5vc1upHePm3lfT2pqPSvntLgtJ8I/P/s3Q//3eM8bq6fidSTNA4NSgoAZc083qIYT+UGfa3D7tS1htvz5WM5uENk70bFN6PxClZqg4a2Gf9bobm9veq+wDz2C3Yc+W8/ZU9EzovxOBQ6WvgwFxrdBA5834TfbXUTXz/ptA4ORYuxQRU4a3+edH/+cu9rn3yBpNMdG/wUI+GroqyeGfnyw/Xno5iUD2eTTp66HwgMbMxJouiUb/O1+86FGl42kcNhXqlt8JVD8vVNg8MM09HIifT+kw+EuqPyFQcO5+0xd1bt0fJO/IiBHWUvpP8e7fReLH4v/esuqe+mjd2Pz3+/ZnaX/9+vpZ29f4ZLEpcXzWZ1bGYxdsrHoXc7nyAdqXqTbtryzYm6FIZTN39qkS+b5EWazA9i+nrxLJi+XrwevOWeB2/k+j9vy3ZhaxNS4Yp7igElnWAqHNn72j+JdljagECW0m0UoTjTPKFZGJpt97ADPoYnRhMfQy109+dm6kaziPfaJ96ISS8nOcA3YriDw32m05QblkLp+cLK7Tx9leVPGo42Q/28BSFlbBfU5j72kPEx5Ev+VIQ0lsgxwfokmvF81xYfK2EkWJMI+Z0GJBWP1LgnpgcyTfRCohbbebD3ePqAdkRedIJiNiZ8stMMoccQL04Xok6la1ErHV4rE5nRK8vaFPHSvArbjiMWmBrJCJlrCb8EqJoziojXdOEorDcos6KIjzcmm8fYcOdSef70Tcavdpuj+8EdTHsBbUL4tSKqo0G46S06qXqFBl28Y5GXSO1IClCYSJR16oiaVQ6AA2mvqhx6P1/aPtnTDj8GBB/f9kCzSleHVP7emGc7FZ/khl/n6phpHAhD6WxVfc03W03rO+rZy8/Np/Gk9ZN5/tpBoqFJzvpWcJs2kjo7O/tXAAAA//+iJmrR" } diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json index 42b6aae2c327..c42b09b0178f 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json @@ -4,16 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * params.multiplier; ctx.zeek.files.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * 1000; ctx.zeek.files.remove('ts');" } }, { "script": { "lang": "painless", "source": "ctx.zeek.session_id = ctx.zeek.files.session_ids[0];", + "if": "ctx.zeek.files.session_ids != null", "ignore_failure": true } }, @@ -34,8 +32,8 @@ { "script": { "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-files\"", - "ignore_failure": true + "source": "ctx.event.id = ctx['zeek']['session_id'];", + "if": "ctx.zeek.session_id != null" } } ] diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json index f2c9f37d4602..c1f0c949f211 100644 --- a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json @@ -3,7 +3,7 @@ "@timestamp": 1547688796000, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.files", - "event.id": "C8I0zn3r9EPbfLgta6-files", + "event.id": "C8I0zn3r9EPbfLgta6", "event.module": "zeek", "fileset.name": "files", "input.type": "log", @@ -41,7 +41,7 @@ "@timestamp": 1547688801000, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.files", - "event.id": "C6sjVo23iNApLnlAt6-files", + "event.id": "C6sjVo23iNApLnlAt6", "event.module": "zeek", "fileset.name": "files", "input.type": "log", diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json index 932224219160..db2d6d2bac67 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json @@ -4,17 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * params.multiplier; ctx.zeek.http.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * 1000; ctx.zeek.http.remove('ts');" } }, { "script": { "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-http\"", - "ignore_failure": true + "source": "ctx.event.id = ctx['zeek']['session_id'];", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json index d47d043d9af6..075b2e2cd023 100644 --- a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json @@ -10,7 +10,7 @@ "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.http", - "event.id": "CCNp8v1SNzY7v9d1Ih-http", + "event.id": "CCNp8v1SNzY7v9d1Ih", "event.module": "zeek", "fileset.name": "http", "http.request.body.bytes": 0, diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml index 9cffd381595a..c722a1b8c2fa 100644 --- a/x-pack/filebeat/module/zeek/notice/config/notice.yml +++ b/x-pack/filebeat/module/zeek/notice/config/notice.yml @@ -18,93 +18,60 @@ processors: - from: "zeek.notice.src" to: "source.address" - when: - has_fields: ["zeek.notice.src"] - from: "zeek.notice.dest" to: "destination.address" - when: - has_fields: ["zeek.notice.dest"] - from: "zeek.notice.uid" to: "zeek.session_id" - when: - has_fields: ["zeek.notice.uid"] - from: "zeek.notice.p" to: "destination.port" - when: - has_fields: ["zeek.notice.p"] - from: "zeek.notice.conn" to: "zeek.notice.connnection_id" - when: - has_fields: ["zeek.notice.conn"] - from: "zeek.notice.iconn" to: "zeek.notice.icmp_id" - when: - has_fields: ["zeek.notice.iconn"] - from: "zeek.notice.id.orig_h" to: "source.address" - when: - has_fields: ["zeek.notice.id.orig_h"] - from: "zeek.notice.id.orig_p" to: "source.port" - when: - has_fields: ["zeek.notice.id.orig_p"] - from: "zeek.notice.id.resp_h" to: "destination.address" - when: - has_fields: ["zeek.notice.id.resp_h"] - from: "zeek.notice.id.resp_p" to: "destination.port" - when: - has_fields: ["zeek.notice.id.resp_p"] - from: "zeek.notice.proto" to: "network.transport" - when: - has_fields: ["zeek.notice.proto"] - from: "zeek.notice.id.orig_p" to: "source.port" - when: - has_fields: ["zeek.notice.id.orig_p"] - from: "zeek.notice.f.id" to: "zeek.notice.file.id" - when: - has_fields: ["zeek.notice.f.id"] - from: "zeek.notice.f.parent_id" to: "dzeek.notice.file.parent_id" - when: - has_fields: ["zeek.notice.f.parent_id"] - from: "zeek.notice.f.source" to: "zeek.notice.file.source" - when: - has_fields: ["zeek.notice.f.source"] - from: "zeek.notice.f.is_orig" to: "zeek.notice.file.is_orig" - when: - has_fields: ["zeek.notice.f.is_orig"] - from: "zeek.notice.f.seen_bytes" to: "zeek.notice.file.seen_bytes" - when: - has_fields: ["zeek.notice.f.seen_bytes"] - from: "zeek.notice.f.total_bytes" to: "zeek.notice.file.total_bytes" - when: - has_fields: ["zeek.notice.f.total_bytes"] + + - from: "zzeek.notice.file_mime_type" + to: "zeek.notice.file.mime_type" ignore_missing: true fail_on_error: false diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json index e41e1a0ad921..80ca5001b6a1 100644 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json @@ -4,29 +4,28 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * params.multiplier; ctx.zeek.notice.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * 1000; ctx.zeek.notice.remove('ts');" } }, { "script": { "lang": "painless", - "source": "if (ctx['destination'] != null && ctx['destination']['address'] != null) { ctx.destination.ip = ctx['destination']['address']; }" + "source": "ctx.destination.ip = ctx['destination']['address'];", + "if": "ctx.destination?.address != null" } }, { "script": { "lang": "painless", - "source": "if (ctx['source'] != null && ctx['source']['address'] != null) { ctx.source.ip = ctx['source']['address']; }" + "source": "ctx.source.ip = ctx['source']['address'];", + "if": "ctx.source?.address != null" } }, { "script": { "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-notice\";", - "ignore_failure": true + "source": "ctx.event.id = ctx['zeek']['session_id'];", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json index 26bfbcb4e348..aab984d1d360 100644 --- a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json @@ -3,7 +3,6 @@ "@timestamp": 1320435875000, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.notice", - "event.id": "null-notice", "event.module": "zeek", "fileset.name": "notice", "input.type": "log", diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json index de32cf75099b..57dea6361994 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json @@ -4,17 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['ssl']['ts'] * params.multiplier; ctx.zeek.ssl.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['ssl']['ts'] * 1000; ctx.zeek.ssl.remove('ts');" } }, { "script": { "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-ssl\"", - "ignore_failure": true + "source": "ctx.event.id = ctx['zeek']['session_id'];", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json index e61c0b33f645..6a034c1d938c 100644 --- a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json @@ -13,7 +13,7 @@ "destination.port": 9243, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.ssl", - "event.id": "CAOvs1BMFCX2Eh0Y3-ssl", + "event.id": "CAOvs1BMFCX2Eh0Y3", "event.module": "zeek", "fileset.name": "ssl", "input.type": "log", @@ -56,7 +56,7 @@ "destination.port": 9243, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.ssl", - "event.id": "C3mki91FnnNtm0u1ok-ssl", + "event.id": "C3mki91FnnNtm0u1ok", "event.module": "zeek", "fileset.name": "ssl", "input.type": "log", From 3f1be8506e3e13357f8d14541b495b9884e1a7b5 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 25 Feb 2019 16:10:21 -0800 Subject: [PATCH 07/16] Rename visualizations for Kibana dashboards --- .../7/dashboard/Filebeat-Zeek-Overview.json | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json index b8d9553330fd..8e12b26cb264 100644 --- a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json +++ b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json @@ -174,8 +174,8 @@ } ], "type": "dashboard", - "updated_at": "2019-02-23T01:54:58.557Z", - "version": "WzE5MjksNF0=" + "updated_at": "2019-02-23T05:05:18.205Z", + "version": "WzMxMTYsNF0=" }, { "attributes": { @@ -190,7 +190,7 @@ } } }, - "title": "Destination Geo [SIEM Zeek]", + "title": "Destination Geo [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -253,7 +253,7 @@ } } }, - "title": "Destination Geo [SIEM Zeek]", + "title": "Destination Geo [Zeek]", "type": "tile_map" } }, @@ -269,8 +269,8 @@ } ], "type": "visualization", - "updated_at": "2019-02-23T01:47:19.123Z", - "version": "WzE5MjEsNF0=" + "updated_at": "2019-02-26T00:06:27.634Z", + "version": "WzMyNzUsNV0=" }, { "attributes": { @@ -285,7 +285,7 @@ } } }, - "title": "Network Transport [SIEM Zeek]", + "title": "Network Transport [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -337,7 +337,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Transport [SIEM Zeek]", + "title": "Network Transport [Zeek]", "type": "pie" } }, @@ -353,8 +353,8 @@ } ], "type": "visualization", - "updated_at": "2019-02-23T01:48:28.840Z", - "version": "WzE5MjIsNF0=" + "updated_at": "2019-02-26T00:07:08.521Z", + "version": "WzMyNzgsNV0=" }, { "attributes": { @@ -369,7 +369,7 @@ } } }, - "title": "Network Application [SIEM Zeek]", + "title": "Network Application [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -421,7 +421,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Application [SIEM Zeek]", + "title": "Network Application [Zeek]", "type": "pie" } }, @@ -437,8 +437,8 @@ } ], "type": "visualization", - "updated_at": "2019-02-23T01:49:36.725Z", - "version": "WzE5MjMsNF0=" + "updated_at": "2019-02-26T00:06:41.868Z", + "version": "WzMyNzYsNV0=" }, { "attributes": { @@ -453,7 +453,7 @@ } } }, - "title": "Network Traffic Direction [SIEM Zeek]", + "title": "Network Traffic Direction [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -505,7 +505,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Traffic Direction [SIEM Zeek]", + "title": "Network Traffic Direction [Zeek]", "type": "pie" } }, @@ -521,8 +521,8 @@ } ], "type": "visualization", - "updated_at": "2019-02-23T01:50:27.347Z", - "version": "WzE5MjQsNF0=" + "updated_at": "2019-02-26T00:06:55.885Z", + "version": "WzMyNzcsNV0=" }, { "attributes": { @@ -537,7 +537,7 @@ } } }, - "title": "Top DNS Domains [SIEM Zeek]", + "title": "Top DNS Domains [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -589,7 +589,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top DNS Domains [SIEM Zeek]", + "title": "Top DNS Domains [Zeek]", "type": "pie" } }, @@ -605,8 +605,8 @@ } ], "type": "visualization", - "updated_at": "2019-02-23T01:51:47.223Z", - "version": "WzE5MjUsNF0=" + "updated_at": "2019-02-26T00:07:23.763Z", + "version": "WzMyNzksNV0=" }, { "attributes": { @@ -621,7 +621,7 @@ } } }, - "title": "Top URL Domains [SIEM Zeek]", + "title": "Top URL Domains [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -688,7 +688,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top URL Domains [SIEM Zeek]", + "title": "Top URL Domains [Zeek]", "type": "pie" } }, @@ -704,8 +704,8 @@ } ], "type": "visualization", - "updated_at": "2019-02-23T01:52:58.606Z", - "version": "WzE5MjYsNF0=" + "updated_at": "2019-02-26T00:07:49.910Z", + "version": "WzMyODEsNV0=" }, { "attributes": { @@ -720,7 +720,7 @@ } } }, - "title": "Top SSL Servers [SIEM Zeek]", + "title": "Top SSL Servers [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -772,7 +772,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top SSL Servers [SIEM Zeek]", + "title": "Top SSL Servers [Zeek]", "type": "pie" } }, @@ -788,8 +788,8 @@ } ], "type": "visualization", - "updated_at": "2019-02-23T01:53:54.810Z", - "version": "WzE5MjcsNF0=" + "updated_at": "2019-02-26T00:07:36.653Z", + "version": "WzMyODAsNV0=" }, { "attributes": { @@ -803,7 +803,7 @@ } } }, - "title": "Time Series Count [SIEM Zeek]", + "title": "Number of Sessions Overtime [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -841,7 +841,7 @@ "time_field": "@timestamp", "type": "timeseries" }, - "title": "Time Series Count [SIEM Zeek]", + "title": "Number of Sessions Overtime [Zeek]", "type": "metrics" } }, @@ -851,8 +851,8 @@ }, "references": [], "type": "visualization", - "updated_at": "2019-02-23T01:54:35.921Z", - "version": "WzE5MjgsNF0=" + "updated_at": "2019-02-26T00:05:56.379Z", + "version": "WzMyNzQsNV0=" } ], "version": "7.0.0-beta1" From 904f1bce3d195556d4028b430d716bdb6d0699b7 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 25 Feb 2019 16:28:31 -0800 Subject: [PATCH 08/16] Add ingest processor code to add tags for connection local_orig and local_resp --- .../module/zeek/connection/ingest/pipeline.json | 14 ++++++++++++++ .../test/connection-json.log-expected.json | 4 +++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json index 3393f112971a..287502de67b5 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json @@ -30,6 +30,20 @@ "if": "ctx.zeek.session_id != null" } }, + { + "script": { + "lang": "painless", + "source": "if (ctx.zeek.connection.local_orig) ctx.tags.add(\"local_orig\");", + "if": "ctx.zeek.connection.local_orig != null" + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx.zeek.connection.local_resp) ctx.tags.add(\"local_resp\");", + "if": "ctx.zeek.connection.local_resp != null" + } + }, { "set": { "field": "source.ip", diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json index ce845d36c809..14be53f52112 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json @@ -24,7 +24,9 @@ "source.packets": 1, "source.port": 38339, "tags": [ - "zeek.connection" + "zeek.connection", + "local_orig", + "local_resp" ], "zeek.connection.history": "Dd", "zeek.connection.local_orig": true, From 43e416bbfdc4ffbecacb8d7f0b427c382d591dfe Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 25 Feb 2019 16:31:13 -0800 Subject: [PATCH 09/16] Add pull to CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 321e83d31f63..bf8001ed88de 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -147,6 +147,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Cover empty request data, url and version in Apache2 module{pull}10730[10730] - Fix registry entries not being cleaned due to race conditions. {pull}10747[10747] - Improve detection of file deletion on Windows. {pull}10747[10747] +- Fix errors in filebeat Zeek dashboard and README files. Add notice.log support. {pull}10916[10916] *Heartbeat* From 5c6b8fc1d304edd99376dce453f09a6881d00d4f Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 25 Feb 2019 16:45:39 -0800 Subject: [PATCH 10/16] Add two more test cases for connection logs related to tags --- .../zeek/connection/test/connection-json.log | 2 + .../test/connection-json.log-expected.json | 81 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log b/x-pack/filebeat/module/zeek/connection/test/connection-json.log index 9e4b15b535ab..4e47ebedceca 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log @@ -1 +1,3 @@ {"ts":1547188415.857497,"uid":"CAcJw21BbVedgFnYH3","id.orig_h":"192.168.86.167","id.orig_p":38339,"id.resp_h":"192.168.86.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":true,"local_resp":true,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]} +{"ts":1547188416.857497,"uid":"CAcJw21BbVedgFnYH4","id.orig_h":"192.168.86.167","id.orig_p":38340,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]} +{"ts":1547188417.857497,"uid":"CAcJw21BbVedgFnYH5","id.orig_h":"4.4.2.2","id.orig_p":383341,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":false,"local_resp":false,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]} diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json index 14be53f52112..becb63faad1b 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json @@ -34,5 +34,86 @@ "zeek.connection.missed_bytes": 0, "zeek.connection.state": "SF", "zeek.session_id": "CAcJw21BbVedgFnYH3" + }, + { + "@timestamp": 1547188416000, + "destination.address": "8.8.8.8", + "destination.bytes": 206, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.packets": 1, + "destination.port": 53, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.connection", + "event.duration": 0.0, + "event.id": "CAcJw21BbVedgFnYH4", + "event.module": "zeek", + "fileset.name": "connection", + "input.type": "log", + "log.offset": 398, + "network.application": "dns", + "network.direction": "outbound", + "network.transport": "udp", + "service.type": "zeek", + "source.address": "192.168.86.167", + "source.bytes": 103, + "source.ip": "192.168.86.167", + "source.packets": 1, + "source.port": 38340, + "tags": [ + "zeek.connection", + "local_orig" + ], + "zeek.connection.history": "Dd", + "zeek.connection.local_orig": true, + "zeek.connection.local_resp": false, + "zeek.connection.missed_bytes": 0, + "zeek.connection.state": "SF", + "zeek.session_id": "CAcJw21BbVedgFnYH4" + }, + { + "@timestamp": 1547188417000, + "destination.address": "8.8.8.8", + "destination.bytes": 206, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.packets": 1, + "destination.port": 53, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.connection", + "event.duration": 0.0, + "event.id": "CAcJw21BbVedgFnYH5", + "event.module": "zeek", + "fileset.name": "connection", + "input.type": "log", + "log.offset": 792, + "network.application": "dns", + "network.direction": "external", + "network.transport": "udp", + "service.type": "zeek", + "source.address": "4.4.2.2", + "source.bytes": 103, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "4.4.2.2", + "source.packets": 1, + "source.port": 383341, + "tags": [ + "zeek.connection" + ], + "zeek.connection.history": "Dd", + "zeek.connection.local_orig": false, + "zeek.connection.local_resp": false, + "zeek.connection.missed_bytes": 0, + "zeek.connection.state": "SF", + "zeek.session_id": "CAcJw21BbVedgFnYH5" } ] \ No newline at end of file From 93a8c61bc24ff4e2ab44d51a30ed6da6b145d18e Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Tue, 26 Feb 2019 08:31:28 -0800 Subject: [PATCH 11/16] Use set processors with if conditions --- .../zeek/connection/ingest/pipeline.json | 6 +++--- .../module/zeek/dns/ingest/pipeline.json | 6 +++--- .../module/zeek/files/ingest/pipeline.json | 6 +++--- .../module/zeek/http/ingest/pipeline.json | 6 +++--- .../module/zeek/notice/ingest/pipeline.json | 18 +++++++++--------- .../module/zeek/ssl/ingest/pipeline.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json index 287502de67b5..8057a09c0672 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json @@ -24,9 +24,9 @@ } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx['zeek']['session_id'];", + "set": { + "field": "ctx.event.id", + "value": "{{ctx.zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json index b4780fae27aa..ea6b17b44c16 100644 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json @@ -8,9 +8,9 @@ } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx['zeek']['session_id'];", + "set": { + "field": "ctx.event.id", + "value": "{{ctx.zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json index c42b09b0178f..45877fcbebb6 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json @@ -30,9 +30,9 @@ } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx['zeek']['session_id'];", + "set": { + "field": "ctx.event.id", + "value": "{{ctx.zeek.session_id}}", "if": "ctx.zeek.session_id != null" } } diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json index db2d6d2bac67..c124b3d2b2db 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json @@ -8,9 +8,9 @@ } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx['zeek']['session_id'];", + "set": { + "field": "ctx.event.id", + "value": "{{ctx.zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json index 80ca5001b6a1..e6db4c99b908 100644 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json @@ -8,23 +8,23 @@ } }, { - "script": { - "lang": "painless", - "source": "ctx.destination.ip = ctx['destination']['address'];", + "set": { + "field": "ctx.destination.ip", + "value": "{{ctx.destination.address}}", "if": "ctx.destination?.address != null" } }, { - "script": { - "lang": "painless", - "source": "ctx.source.ip = ctx['source']['address'];", + "set": { + "field": "ctx.source.ip", + "value": "{{ctx.source.address}}", "if": "ctx.source?.address != null" } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx['zeek']['session_id'];", + "set": { + "field": "ctx.event.id", + "value": "{{ctx.zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json index 57dea6361994..d41cf0fa8aa6 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json @@ -8,9 +8,9 @@ } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx['zeek']['session_id'];", + "set": { + "field": "ctx.event.id", + "value": "{{ctx.zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, From 95d72cd5cd9721a48a515516eb70c6c9c4f1fc74 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Tue, 26 Feb 2019 09:10:43 -0800 Subject: [PATCH 12/16] Merge with upstream/master --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index a119227ca519..0d0fd4fbc2a2 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] retries: 300 @@ -16,7 +16,7 @@ services: - "xpack.security.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.1.0-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.0.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -26,7 +26,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.1.0-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.0.0-SNAPSHOT healthcheck: test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);'''] retries: 600 From 6c2ad76668752c8ecd1fed88e2a58f87fa149f1b Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Tue, 26 Feb 2019 11:47:31 -0800 Subject: [PATCH 13/16] Fix copy/paste errors --- testing/environments/snapshot.yml | 6 +++--- .../zeek/connection/ingest/pipeline.json | 4 ++-- .../module/zeek/dns/ingest/pipeline.json | 4 ++-- .../module/zeek/files/ingest/pipeline.json | 4 ++-- .../module/zeek/http/ingest/pipeline.json | 4 ++-- .../module/zeek/notice/ingest/pipeline.json | 12 +++++------ .../module/zeek/ssl/ingest/pipeline.json | 4 ++-- x-pack/filebeat/modules.d/zeek.yml.disabled | 21 ------------------- 8 files changed, 19 insertions(+), 40 deletions(-) delete mode 100644 x-pack/filebeat/modules.d/zeek.yml.disabled diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 0d0fd4fbc2a2..a119227ca519 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] retries: 300 @@ -16,7 +16,7 @@ services: - "xpack.security.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.0.0-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.1.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -26,7 +26,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.0.0-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.1.0-SNAPSHOT healthcheck: test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);'''] retries: 600 diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json index 8057a09c0672..1ca5eadc4099 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json @@ -25,8 +25,8 @@ }, { "set": { - "field": "ctx.event.id", - "value": "{{ctx.zeek.session_id}}", + "field": "event.id", + "value": "{{zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json index ea6b17b44c16..bea3798a7bbd 100644 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json @@ -9,8 +9,8 @@ }, { "set": { - "field": "ctx.event.id", - "value": "{{ctx.zeek.session_id}}", + "field": "event.id", + "value": "{{zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json index 45877fcbebb6..84e96dbd912e 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json @@ -31,8 +31,8 @@ }, { "set": { - "field": "ctx.event.id", - "value": "{{ctx.zeek.session_id}}", + "field": "event.id", + "value": "{{zeek.session_id}}", "if": "ctx.zeek.session_id != null" } } diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json index c124b3d2b2db..a892d959ce5a 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json @@ -9,8 +9,8 @@ }, { "set": { - "field": "ctx.event.id", - "value": "{{ctx.zeek.session_id}}", + "field": "event.id", + "value": "{{zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json index e6db4c99b908..1b1bf8b49af0 100644 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json @@ -9,22 +9,22 @@ }, { "set": { - "field": "ctx.destination.ip", - "value": "{{ctx.destination.address}}", + "field": "destination.ip", + "value": "{{destination.address}}", "if": "ctx.destination?.address != null" } }, { "set": { - "field": "ctx.source.ip", - "value": "{{ctx.source.address}}", + "field": "source.ip", + "value": "{{source.address}}", "if": "ctx.source?.address != null" } }, { "set": { - "field": "ctx.event.id", - "value": "{{ctx.zeek.session_id}}", + "field": "event.id", + "value": "{{zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json index d41cf0fa8aa6..54d068b19f9b 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json @@ -9,8 +9,8 @@ }, { "set": { - "field": "ctx.event.id", - "value": "{{ctx.zeek.session_id}}", + "field": "event.id", + "value": "{{zeek.session_id}}", "if": "ctx.zeek.session_id != null" } }, diff --git a/x-pack/filebeat/modules.d/zeek.yml.disabled b/x-pack/filebeat/modules.d/zeek.yml.disabled deleted file mode 100644 index c43668021eab..000000000000 --- a/x-pack/filebeat/modules.d/zeek.yml.disabled +++ /dev/null @@ -1,21 +0,0 @@ -# Module: zeek -# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-zeek.html - -- module: zeek - # All logs - connection: - enabled: true - dns: - enabled: true - http: - enabled: true - files: - enabled: true - ssl: - enabled: true - notice: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: From 31412e5aec3404fa0c2e0da0bd7bfd7aebe7e3ce Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Tue, 26 Feb 2019 11:53:28 -0800 Subject: [PATCH 14/16] Fix copy/paste errors --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index a119227ca519..0d0fd4fbc2a2 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] retries: 300 @@ -16,7 +16,7 @@ services: - "xpack.security.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.1.0-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.0.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -26,7 +26,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.1.0-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.0.0-SNAPSHOT healthcheck: test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);'''] retries: 600 From e73fce239a65a6d77b6a829625b10dbdd1a3999e Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Tue, 26 Feb 2019 22:52:53 -0800 Subject: [PATCH 15/16] Fix a typo in Suricata eve pipeline that caused test to fail --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 3276c1968db7..49266572b2b9 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -64,7 +64,7 @@ }, { "script": { - "type": "painless", + "lang": "painless", "source": "def domain = ctx.destination?.domain; if (domain instanceof Collection) { domain = domain.stream().distinct().collect(Collectors.toList()); if (domain.length == 1) { domain = domain[0]; }ctx.destination.domain = domain; }", "ignore_failure": true } From e8cbbfec6e2673cd343df7e392c5927b15f9f4f2 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Wed, 27 Feb 2019 07:54:43 -0800 Subject: [PATCH 16/16] Leave snapshot at 7.1.0, will have a new PR to update it to 8.0.0 --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 0d0fd4fbc2a2..a119227ca519 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] retries: 300 @@ -16,7 +16,7 @@ services: - "xpack.security.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.0.0-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.1.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -26,7 +26,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.0.0-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.1.0-SNAPSHOT healthcheck: test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);'''] retries: 600