[Filebeat] Replace Suricata/Eve fields with aliases to ECS fields (#1…

…0377)

This PR avoids duplicate data in documents ingested via the Suricata Eve fileset by replacing a few fields with aliases to ECS fields.

This allows to maintain the full set of fields Suricata users may expect while at the same time reducing the size of the events.

Also, the following non-related fixes are performed:
- "http.response.status_code" was being set from a string and not an integer.
- "user_agent.original" was being inadvertently removed by the user_agent processor.
- Reformatted the ingest pipeline with standard JSON formatting.

dev-tools/ecs-migration.yml

@@ -171,6 +171,126 @@
   alias: true
   beat: filebeat
 
+- from: suricata.eve.alert.action
+  to: event.outcome
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.alert.severity
+  to: event.severity
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.app_proto
+  to: network.protocol
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.dest_ip
+  to: destination.ip
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.dest_port
+  to: destination.port
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.event_type
+  to: event.type
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.fileinfo.filename
+  to: file.path
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.fileinfo.size
+  to: file.size
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.flow.start
+  to: event.start
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.flow.bytes_toclient
+  to: destination.bytes
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.flow.bytes_toserver
+  to: source.bytes
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.flow.pkts_toclient
+  to: destination.packets
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.flow.pkts_toserver
+  to: source.packets
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.http.hostname
+  to: url.domain
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.http.http_method
+  to: http.request.method
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.http.http_refer
+  to: http.request.referrer
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.http.http_user_agent
+  to: user_agent.original
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.http.length
+  to: http.response.body.bytes
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.http.status
+  to: http.response.status_code
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.http.url
+  to: url.original
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.proto
+  to: network.transport
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.src_ip
+  to: source.ip
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.src_port
+  to: source.port
+  alias: true
+  beat: filebeat
+
+- from: suricata.eve.timestamp
+  to: '@timestamp'
+  alias: true
+  beat: filebeat
+
 ## System module
 
 - from: system.syslog.hostname

filebeat/docs/fields.asciidoc

@@ -11038,7 +11038,9 @@ Fields exported by the EVE JSON logs
 *`suricata.eve.event_type`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: event.type
 
 --
 
@@ -11124,7 +11126,9 @@ type: keyword
 *`suricata.eve.fileinfo.filename`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: file.path
 
 --
 
@@ -11173,7 +11177,9 @@ type: keyword
 *`suricata.eve.fileinfo.size`*::
 +
 --
-type: long
+type: alias
+
+alias to: file.size
 
 --
 
@@ -11187,21 +11193,27 @@ type: long
 *`suricata.eve.dest_port`*::
 +
 --
-type: long
+type: alias
+
+alias to: destination.port
 
 --
 
 *`suricata.eve.src_port`*::
 +
 --
-type: long
+type: alias
+
+alias to: source.port
 
 --
 
 *`suricata.eve.proto`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: network.transport
 
 --
 
@@ -11215,7 +11227,9 @@ type: long
 *`suricata.eve.src_ip`*::
 +
 --
-type: ip
+type: alias
+
+alias to: source.ip
 
 --
 
@@ -11294,7 +11308,9 @@ type: keyword
 *`suricata.eve.dest_ip`*::
 +
 --
-type: ip
+type: alias
+
+alias to: destination.ip
 
 --
 
@@ -11309,7 +11325,9 @@ type: long
 *`suricata.eve.http.status`*::
 +
 --
-type: long
+type: alias
+
+alias to: http.response.status_code
 
 --
 
@@ -11323,7 +11341,9 @@ type: keyword
 *`suricata.eve.http.http_user_agent`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: user_agent.original
 
 --
 
@@ -11337,35 +11357,45 @@ type: keyword
 *`suricata.eve.http.http_refer`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: http.request.referrer
 
 --
 
 *`suricata.eve.http.url`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: url.original
 
 --
 
 *`suricata.eve.http.hostname`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: url.domain
 
 --
 
 *`suricata.eve.http.length`*::
 +
 --
-type: long
+type: alias
+
+alias to: http.response.body.bytes
 
 --
 
 *`suricata.eve.http.http_method`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: http.request.method
 
 --
 
@@ -11379,7 +11409,9 @@ type: keyword
 *`suricata.eve.timestamp`*::
 +
 --
-type: date
+type: alias
+
+alias to: @timestamp
 
 --
 
@@ -11401,7 +11433,9 @@ type: keyword
 *`suricata.eve.alert.severity`*::
 +
 --
-type: long
+type: alias
+
+alias to: event.severity
 
 --
 
@@ -11429,7 +11463,9 @@ type: keyword
 *`suricata.eve.alert.action`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: event.outcome
 
 --
 
@@ -12377,21 +12413,27 @@ type: keyword
 *`suricata.eve.flow.bytes_toclient`*::
 +
 --
-type: long
+type: alias
+
+alias to: destination.bytes
 
 --
 
 *`suricata.eve.flow.start`*::
 +
 --
-type: date
+type: alias
+
+alias to: event.start
 
 --
 
 *`suricata.eve.flow.pkts_toclient`*::
 +
 --
-type: long
+type: alias
+
+alias to: destination.packets
 
 --
 
@@ -12412,7 +12454,9 @@ type: keyword
 *`suricata.eve.flow.bytes_toserver`*::
 +
 --
-type: long
+type: alias
+
+alias to: source.bytes
 
 --
 
@@ -12426,7 +12470,9 @@ type: keyword
 *`suricata.eve.flow.pkts_toserver`*::
 +
 --
-type: long
+type: alias
+
+alias to: source.packets
 
 --
 
@@ -12447,7 +12493,9 @@ type: boolean
 *`suricata.eve.app_proto`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: network.protocol
 
 --
 

x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/Filebeat-Suricata-Alert-Overview.json

@@ -226,10 +226,10 @@
         "columns": [
           "host.name",
           "suricata.eve.flow_id",
-          "suricata.eve.src_ip",
-          "suricata.eve.src_port",
-          "suricata.eve.dest_ip",
-          "suricata.eve.dest_port",
+          "source.ip",
+          "source.port",
+          "destination.ip",
+          "destination.port",
           "source.geo.country_iso_code",
           "destination.geo.country_iso_code"
         ],