From feb3a9636232dbdc2213745093b4374997d8b5a1 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 24 Jan 2019 09:37:00 +0100 Subject: [PATCH] Cleanup ecs-migration.yml file (#10278) This adds the beat name it applies to to each entry. This will allow us to create a migration table for each Beat instead of just having all the fields. It's also potentially relevant for the dashboard migration as a field might be migrated in one beat but not an other. Further changes: * Extend all geo.* items to specify each migrated field. This is needed for the dashboard migration * Extend all user_agent.* items to specify each migrated field. This is needed for the dashboard migration * Clean up broken entry for resolve.host * Add Beat name to each entry --- dev-tools/ecs-migration.yml | 405 +++++++++++++++++++++++++++++++++--- 1 file changed, 371 insertions(+), 34 deletions(-) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index e52512d2046..1a8c523d934 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -12,28 +12,10 @@ # alias6: true-if-alias-is-required-in-6 # # Copy to is useful for fields where multiple fields map to the same ECS field # copy_to: true-if-field-should-be-copied-to-target-in-6x +# # Beat name this applies to, if not beat is set, applies to all Beats +# beat: filebeat -- from: offset - to: log.offset - alias6: true - alias: true - -- from: fileset.name - to: event.dataset - alias6: true - alias: true - -- from: fileset.module - to: event.module - alias6: true - alias: true - -- from: source - to: - - log.file.path - - log.source.address - alias: false - +## Libbeat - from: beat.name to: host.name alias6: true @@ -54,7 +36,6 @@ alias6: true alias: true - # Processor fields # Docker processor @@ -119,6 +100,33 @@ to: process.executable alias: true + +## Filebeat +- from: offset + to: log.offset + alias6: true + alias: true + beat: filebeat + +- from: fileset.name + to: event.dataset + alias6: true + alias: true + beat: filebeat + +- from: fileset.module + to: event.module + alias6: true + alias: true + beat: filebeat + +- from: source + to: + - log.file.path + - log.source.address + alias: false + beat: filebeat + # Filebeat modules # Suricata module @@ -126,183 +134,315 @@ - from: source_ecs.ip to: source.ip alias: true + beat: filebeat - from: source_ecs.port to: source.port alias: true + beat: filebeat - from: source_ecs.geo.continent_name to: source.geo.continent_name alias: true + beat: filebeat - from: source_ecs.geo.country_iso_code to: source.geo.country_iso_code alias: true + beat: filebeat - from: source_ecs.geo.location to: source.geo.location alias: true + beat: filebeat - from: source_ecs.geo.region_name to: source.geo.region_name alias: true + beat: filebeat - from: source_ecs.geo.city_name to: source.geo.city_name alias: true + beat: filebeat - from: source_ecs.geo.region_iso_code to: source.geo.region_iso_code alias: true + beat: filebeat ## System module - from: system.syslog.hostname to: host.hostname alias: true + beat: filebeat - from: system.syslog.program to: process.name alias: true + beat: filebeat - from: system.syslog.pid to: process.pid alias: true + beat: filebeat - from: system.syslog.message to: message alias: true + beat: filebeat - from: system.auth.hostname to: host.hostname alias: true copy_to: false + beat: filebeat - from: system.auth.pid to: process.pid alias: true + beat: filebeat - from: system.auth.groupadd.gid to: group.id alias: true + beat: filebeat - from: system.auth.useradd.uid to: user.id alias: true + beat: filebeat - from: system.auth.useradd.user to: user.name alias: true + beat: filebeat - from: system.auth.ssh.event to: event.action alias: true + beat: filebeat - from: system.auth.program to: process.name alias: true + beat: filebeat - from: system.auth.ssh.ip to: source.ip alias: true + beat: filebeat - from: system.auth.ssh.port to: source.port alias: true + beat: filebeat + +- from: system.auth.ssh.geoip.continent_name + to: source.geo.continent_name + alias: true + beat: filebeat + +- from: system.auth.ssh.geoip.country_iso_code + to: source.geo.country_iso_code + alias: true + beat: filebeat + +- from: system.auth.ssh.geoip.location + to: source.geo.location + alias: true + beat: filebeat + +- from: system.auth.ssh.geoip.region_name + to: source.geo.region_name + alias: true + beat: filebeat + +- from: system.auth.ssh.geoip.city_name + to: source.geo.city_name + alias: true + beat: filebeat -- from: system.auth.ssh.geoip.* - to: source.geo.* +- from: system.auth.ssh.geoip.region_iso_code + to: source.geo.region_iso_code alias: true + beat: filebeat ## Apache module - from: apache2.access.remote_ip to: source.address alias: true + beat: filebeat - from: apache2.access.user_name to: user.name alias: true + beat: filebeat - from: apache2.access.method to: http.request.method alias: true + beat: filebeat - from: apache2.access.url to: url.original alias: true + beat: filebeat - from: apache2.access.http_version to: http.version alias: true + beat: filebeat - from: apache2.access.response_code to: http.response.status_code alias: true + beat: filebeat - from: apache2.access.referrer to: http.request.referrer alias: true + beat: filebeat - from: apache2.access.agent to: user_agent.original alias: true + beat: filebeat - from: apache2.access.body_sent.bytes to: http.response.body.bytes alias: true # These expand all fields under geoip and user_agent -- from: apache2.access.geoip.* - to: source.geo.* +- from: apache2.access.geoip.continent_name + to: source.geo.continent_name + alias: true + beat: filebeat + +- from: apache2.access.geoip.country_iso_code + to: source.geo.country_iso_code alias: true + beat: filebeat -- from: apache2.access.user_agent.* - to: user_agent.* +- from: apache2.access.geoip.location + to: source.geo.location alias: true + beat: filebeat + +- from: apache2.access.geoip.region_name + to: source.geo.region_name + alias: true + beat: filebeat + +- from: apache2.access.geoip.city_name + to: source.geo.city_name + alias: true + beat: filebeat + +- from: apache2.access.geoip.region_iso_code + to: source.geo.region_iso_code + alias: true + beat: filebeat + +- from: apache2.access.user_agent.device + to: user_agent.device + alias: true + beat: filebeat + +- from: apache2.access.user_agent.major + to: user_agent.major + alias: true + beat: filebeat + +- from: apache2.access.user_agent.minor + to: user_agent.minor + alias: true + beat: filebeat + +- from: apache2.access.user_agent.name + to: user_agent.name + alias: true + beat: filebeat + +- from: apache2.access.user_agent.original + to: user_agent.original + alias: true + beat: filebeat + +- from: apache2.access.user_agent.os + to: user_agent.os.full_name + alias: true + beat: filebeat + +- from: apache2.access.user_agent.os_name + to: user_agent.os.name + alias: true + beat: filebeat + +- from: apache2.access.user_agent.patch + to: user_agent.patch + alias: true + beat: filebeat + +- from: apache2.access.user_agent.user_name + to: user.name + alias: true + beat: filebeat ### Error fileset - from: apache2.error.message to: message alias: true + beat: filebeat - from: apache2.error.level to: log.level alias: true + beat: filebeat - from: apache2.error.client to: source.address alias: true + beat: filebeat - from: apache2.error.pid to: process.pid alias: true + beat: filebeat - from: apache2.error.tid to: process.thread.id alias: true + beat: filebeat ## Elasticsearch module - from: elasticsearch.audit.event_type to: event.type alias: true + beat: filebeat - from: elasticsearch.audit.origin_address to: source.ip alias: true + beat: filebeat - from: elasticsearch.audit.principal to: user.name alias: true + beat: filebeat - from: elasticsearch.audit.request_body to: http.request.body.content alias: true + beat: filebeat - from: elasticsearch.audit.uri to: url.original alias: true + beat: filebeat - from: elasticsearch.slowlog.took_millis to: event.duration @@ -314,90 +454,112 @@ - from: iis.access.server_ip to: destination.address alias: true + beat: filebeat - from: iis.access.remote_ip to: source.address alias: true + beat: filebeat - from: iis.access.url to: url.path alias: true + beat: filebeat - from: iis.access.query_string to: url.query alias: true + beat: filebeat - from: iis.access.port to: destination.port alias: true + beat: filebeat - from: iis.access.user_name to: user.name alias: true + beat: filebeat - from: iis.access.hostname to: destination.domain alias: true + beat: filebeat - from: iis.access.user_agent.original to: user_agent.original alias: true + beat: filebeat - from: iis.access.geoip.continent_name to: source.geo.continent_name alias: true + beat: filebeat - from: iis.access.geoip.country_iso_code to: source.geo.country_iso_code alias: true + beat: filebeat - from: iis.access.geoip.location to: source.geo.location alias: true + beat: filebeat - from: iis.access.geoip.region_name to: source.geo.region_name alias: true + beat: filebeat - from: iis.access.geoip.city_name to: source.geo.city_name alias: true + beat: filebeat - from: iis.access.geoip.region_iso_code to: source.geo.region_iso_code alias: true + beat: filebeat - from: iis.access.method to: http.request.method alias: true + beat: filebeat - from: iis.access.response_code to: http.response.status_code alias: true + beat: filebeat - from: iis.access.referrer to: http.request.referrer alias: true + beat: filebeat - from: iis.error.remote_ip to: source.address alias: true + beat: filebeat - from: iis.error.remote_port to: source.port alias: true + beat: filebeat - from: iis.error.server_ip to: destination.address alias: true + beat: filebeat - from: iis.error.server_port to: destination.port alias: true + beat: filebeat - from: iis.error.http_version to: http.version alias: true + beat: filebeat - from: iis.access.body_received.bytes to: http.request.body.bytes @@ -415,38 +577,47 @@ - from: iis.error.method to: http.request.method alias: true + beat: filebeat - from: iis.error.url to: url.original alias: true + beat: filebeat - from: iis.error.response_code to: http.response.status_code alias: true + beat: filebeat - from: iis.error.geoip.continent_name to: source.geo.continent_name alias: true + beat: filebeat - from: iis.error.geoip.country_iso_code to: source.geo.country_iso_code alias: true + beat: filebeat - from: iis.error.geoip.location to: source.geo.location alias: true + beat: filebeat - from: iis.error.geoip.region_name to: source.geo.region_name alias: true + beat: filebeat - from: iis.error.geoip.city_name to: source.geo.city_name alias: true + beat: filebeat - from: iis.error.geoip.region_iso_code to: source.geo.region_iso_code alias: true + beat: filebeat ## HAProxy module @@ -463,64 +634,79 @@ - from: haproxy.client.ip to: source.address alias: true + beat: filebeat - from: haproxy.client.port to: source.port alias: true + beat: filebeat - from: haproxy.process_name to: process.name alias: true + beat: filebeat - from: haproxy.pid to: process.pid alias: true + beat: filebeat - from: haproxy.destination.ip to: destination.ip alias: true + beat: filebeat - from: haproxy.destination.port to: destination.port alias: true + beat: filebeat - from: haproxy.http.response.status_code to: http.response.status_code alias: true + beat: filebeat - from: haproxy.geoip.continent_name to: source.geo.continent_name alias: true + beat: filebeat - from: haproxy.geoip.country_iso_code to: source.geo.country_iso_code alias: true + beat: filebeat - from: haproxy.geoip.location to: source.geo.location alias: true + beat: filebeat - from: haproxy.geoip.region_name to: source.geo.region_name alias: true + beat: filebeat - from: haproxy.geoip.city_name to: source.geo.city_name alias: true + beat: filebeat - from: haproxy.geoip.region_iso_code to: source.geo.region_iso_code alias: true + beat: filebeat ## Kafka module - from: kafka.log.level to: log.level alias: true + beat: filebeat - from: kafka.log.message to: message alias: true + beat: filebeat ## Kibana module @@ -544,88 +730,108 @@ - from: kibana.log.meta.req.headers.referer to: http.request.referrer alias: true + beat: filebeat - from: kibana.log.meta.req.referer to: http.request.referrer alias: true + beat: filebeat - from: kibana.log.meta.req.headers.user-agent to: user_agent.original alias: true + beat: filebeat - from: kibana.log.meta.req.remoteAddress to: source.address alias: true + beat: filebeat - from: kibana.log.meta.req.url to: url.original alias: true + beat: filebeat - from: kibana.log.meta.meta.statusCode to: http.response.status_code alias: true + beat: filebeat - from: kibana.log.meta.method to: http.request.method alias: true + beat: filebeat ## Logstash module - from: logstash.log.level to: log.level alias: true + beat: filebeat - from: logstash.log.message to: message alias: true + beat: filebeat - from: logstash.slowlog.level to: log.level alias: true + beat: filebeat - from: logstash.slowlog.took_in_nanos to: event.duration alias: true + beat: filebeat ## MongoDB module - from: mongodb.log.severity to: log.level alias: true + beat: filebeat - from: mongodb.log.message to: message alias: true + beat: filebeat ## MySQL module - from: mysql.error.message to: message alias: true + beat: filebeat - from: mysql.error.level to: log.level alias: true + beat: filebeat - from: mysql.error.thread_id to: mysql.thread_id alias: true + beat: filebeat - from: mysql.slowlog.ip to: source.ip alias: true + beat: filebeat - from: mysql.slowlog.host to: source.domain alias: true + beat: filebeat - from: mysql.slowlog.user to: user.name alias: true + beat: filebeat - from: mysql.slowlog.id to: mysql.thread_id alias: true + beat: filebeat - from: mysql.slowlog.query_time.sec to: event.duration @@ -637,30 +843,37 @@ - from: nginx.access.user_name to: user.name alias: true + beat: filebeat - from: nginx.access.url to: url.original alias: true + beat: filebeat - from: nginx.access.agent to: user_agent.original alias: true + beat: filebeat - from: nginx.access.response_code to: http.response.status_code alias: true + beat: filebeat - from: nginx.access.referrer to: http.request.referrer alias: true + beat: filebeat - from: nginx.access.method to: http.request.method alias: true + beat: filebeat - from: nginx.access.http_version to: http.version alias: true + beat: filebeat - from: nginx.access.body_sent.bytes to: http.response.body.bytes @@ -669,50 +882,62 @@ - from: nginx.access.geoip.continent_name to: source.geo.continent_name alias: true + beat: filebeat - from: nginx.access.geoip.country_iso_code to: source.geo.country_iso_code alias: true + beat: filebeat - from: nginx.access.geoip.location to: source.geo.location alias: true + beat: filebeat - from: nginx.access.geoip.region_name to: source.geo.region_name alias: true + beat: filebeat - from: nginx.access.geoip.city_name to: source.geo.city_name alias: true + beat: filebeat - from: nginx.access.geoip.region_iso_code to: source.geo.region_iso_code alias: true + beat: filebeat - from: nginx.access.agent to: user_agent.original alias: true + beat: filebeat - from: nginx.error.level to: log.level alias: true + beat: filebeat - from: nginx.error.pid to: process.pid alias: true + beat: filebeat - from: nginx.error.tid to: process.thread.id alias: true + beat: filebeat - from: nginx.error.message to: message alias: true + beat: filebeat - from: nginx.access.remote_ip_list to: network.forwarded_ip alias: true + beat: filebeat ## PostgreSQL module @@ -724,125 +949,198 @@ - from: postgresql.log.timezone to: event.timezone alias: true + beat: filebeat - from: postgresql.log.thread_id to: process.pid alias: true + beat: filebeat - from: postgresql.log.user to: user.name alias: true + beat: filebeat - from: postgresql.log.level to: log.level alias: true + beat: filebeat - from: postgresql.log.message to: message alias: true + beat: filebeat ## Redis module - from: redis.log.pid to: process.pid alias: true + beat: filebeat - from: redis.log.level to: log.level alias: true + beat: filebeat - from: redis.log.message to: message alias: true + beat: filebeat ## Icinga module - from: icinga.debug.message to: message alias: true + beat: filebeat + - from: icinga.debug.severity to: log.level alias: true + beat: filebeat - from: icinga.main.message to: message alias: true + beat: filebeat + - from: icinga.main.severity to: log.level alias: true + beat: filebeat - from: icinga.startup.message to: message alias: true + beat: filebeat + - from: icinga.startup.severity to: log.level alias: true + beat: filebeat ## Traefik module - from: traefik.access.remote_ip to: source.address alias: true + beat: filebeat - from: traefik.access.url to: url.original alias: true + beat: filebeat - from: traefik.access.user_name to: user.name alias: true + beat: filebeat - from: traefik.access.agent to: user_agent.original alias: true + beat: filebeat + +- from: traefik.access.user_agent.original + to: user_agent.original + alias: true + beat: filebeat + +- from: traefik.access.user_agent.device + to: user_agent.device + alias: true + beat: filebeat + +- from: traefik.access.user_agent.major + to: user_agent.major + alias: true + beat: filebeat + +- from: traefik.access.user_agent.minor + to: user_agent.minor + alias: true + beat: filebeat + +- from: traefik.access.user_agent.name + to: user_agent.name + alias: true + beat: filebeat - from: traefik.access.user_agent.original to: user_agent.original alias: true + beat: filebeat + +- from: traefik.access.user_agent.os + to: user_agent.os.full_name + alias: true + beat: filebeat + +- from: traefik.access.user_agent.os_name + to: user_agent.os.name + alias: true + beat: filebeat + +- from: traefik.access.user_agent.patch + to: user_agent.patch + alias: true + beat: filebeat -- from: traefik.access.user_agent.* - to: user_agent.* +- from: traefik.access.user_agent.user_name + to: user.name alias: true + beat: filebeat - from: traefik.access.geoip.continent_name to: source.geo.continent_name alias: true + beat: filebeat - from: traefik.access.geoip.country_iso_code to: source.geo.country_iso_code alias: true + beat: filebeat - from: traefik.access.geoip.location to: source.geo.location alias: true + beat: filebeat - from: traefik.access.geoip.region_name to: source.geo.region_name alias: true + beat: filebeat - from: traefik.access.geoip.city_name to: source.geo.city_name alias: true + beat: filebeat - from: traefik.access.geoip.region_iso_code to: source.geo.region_iso_code alias: true + beat: filebeat - from: traefik.access.method to: http.request.method alias: true + beat: filebeat - from: traefik.access.response_code to: http.response.status_code alias: true + beat: filebeat - from: traefik.access.referrer to: http.request.referrer alias: true + beat: filebeat - from: traefik.access.http_version to: http.version alias: true + beat: filebeat - from: traefik.access.body_sent.bytes to: http.response.body.bytes @@ -866,15 +1164,17 @@ - from: source.hostname to: source.domain alias: true + beat: auditbeat - from: process.exe to: process.executable alias: true + beat: auditbeat - from: process.cwd to: process.working_directory alias: true - + beat: auditbeat # Metricbeat @@ -884,26 +1184,30 @@ alias: false comment: Data structure of field changed, old field will stay around breaking: true + beat: metricbeat - from: metricset.module to: event.module alias: true + beat: metricbeat - from: metricset.rrt to: event.duration alias: false comment: Unit changed, old field will stay around if needed breaking: true + beat: metricbeat - from: metricset.host to: service.address alias: true + beat: metricbeat - from: metricset.namespace to: event.dataset alias: false comment: No alias mapping as field did not always exist - + beat: metricbeat # Packetbeat @@ -912,16 +1216,19 @@ to: source.bytes alias: false comment: Don't add an alias until all of Packetbeat stops using this field. + beat: packetbeat - from: bytes_out to: destination.bytes alias: false comment: Don't add an alias until all of Packetbeat stops using this field. + beat: packetbeat - from: notes to: error.message alias: false comment: Don't add an alias until all of Packetbeat stops using this field. + beat: packetbeat - from: responsetime to: event.duration @@ -929,78 +1236,96 @@ comment: > Units changed from usec to nsec. Don't add an alias until all of Packetbeat stops using this field. + beat: packetbeat - from: transport to: network.transport alias: false comment: Don't add an alias until all of Packetbeat stops using this field. + beat: packetbeat ## Flows - from: flow_id to: flow.id alias: true + beat: packetbeat - from: final to: flow.final alias: true + beat: packetbeat - from: start_time to: event.start alias: true + beat: packetbeat - from: last_time to: event.end alias: true + beat: packetbeat - from: source.stats.net_bytes_total to: source.bytes alias: true + beat: packetbeat - from: source.stats.net_packets_total to: source.packets alias: true + beat: packetbeat - from: dest.stats.net_bytes_total to: destination.bytes alias: true + beat: packetbeat - from: dest.stats.net_packets_total to: destination.packets alias: true + beat: packetbeat ## HTTP - from: http.request.body to: http.request.body.content alias: false + beat: packetbeat - from: http.response.body to: http.response.body.content alias: false + beat: packetbeat - from: http.response.code to: http.response.status_code alias: true + beat: packetbeat - from: http.response.phrase to: http.response.status_phrase alias: true + beat: packetbeat - from: http.request.params to: url.query alias: true + beat: packetbeat - from: method to: http.request.method alias: false comment: method is used by other protocols. + beat: packetbeat - from: path to: url.path alias: false + beat: packetbeat - from: real_ip to: network.forwarded_ip alias: false + beat: packetbeat ## MySQL - from: mysql.iserror @@ -1009,6 +1334,7 @@ comment: > Status reflects whether or not an error occured. Its values are either OK or Error. + beat: packetbeat ## NFS - from: rpc.time @@ -1017,6 +1343,7 @@ comment: > Cannot add alias at this time because the field is used by MongoDB which has not been updated yet. + beat: packetbeat - from: rpc.call_size to: source.bytes @@ -1024,6 +1351,7 @@ comment: > Cannot add alias at this time because the field is used by MongoDB which has not been updated yet. + beat: packetbeat - from: rpc.reply_size to: destination.bytes @@ -1031,39 +1359,48 @@ comment: > Cannot add alias at this time because the field is used by MongoDB which has not been updated yet. + beat: packetbeat ## Cassandra - from: no_request to: cassandra.no_request alias: true - + beat: packetbeat # Heartbeat - from: monitor.scheme to: url.scheme alias: true + beat: heartbeat - from: monitor.host to: url.domain alias: true + beat: heartbeat - - from: resolve.host +- from: resolve.host to: url.domain alias: true + beat: heartbeat - from: http.url to: url.full alias: true + beat: heartbeat - from: tcp.port to: url.port alias: true + beat: heartbeat # Journalbeat - from: host.name to: host.hostname alias: false + beat: journalbeat + comment: This field should not be renamed as it would cause issue some Beats and Journalbeat does not have dashboards + rename: false - from: read_timestamp to: event.created