Filebeat modules now use .address fields for ambiguous address value (e…

…lastic#10141) The `source.address` field is used prior to extracting IP or domain, when an event source can contain either. Migrated: * `apache2.access.remote_ip` => `source.address` * Note that the renamed apache module has not been released yet, so the field `apache.access.remote_ip` was simply removed. * `haproxy.client.ip` => `source.address`
andrewkroh · Jan 18, 2019 · 4a837b7 · 4a837b7
1 parent eee0c50
commit 4a837b7
Show file tree

Hide file tree

Showing 15 changed files with 40 additions and 48 deletions.
diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -216,6 +216,10 @@
 
 ## Apache module
 
+- from: apache2.access.remote_ip
+  to: source.address
+  alias: true
+
 - from: apache2.access.user_name
   to: user.name
   alias: true
@@ -424,6 +428,10 @@
 
 ## HAProxy module
 
+- from: haproxy.client.ip
+  to: source.address
+  alias: true
+
 - from: haproxy.client.port
   to: source.port
   alias: true

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -63,7 +63,7 @@ Aliases for backward compatibility with old apache2 fields
 --
 type: alias
 
-alias to: apache.access.remote_ip
+alias to: source.address
 
 --
 
@@ -363,16 +363,6 @@ Contains fields for the Apache HTTP Server access logs.
 
 
 
-*`apache.access.remote_ip`*::
-+
---
-type: keyword
-
-Client IP address or hostname.
-
-
---
-
 *`apache.access.ssl.protocol`*::
 +
 --
@@ -4742,9 +4732,9 @@ Information about the client doing the request
 *`haproxy.client.ip`*::
 +
 --
-IP address of the client which initiated the TCP connection to haproxy.
-If connection is via unix socket, socket path is in this field.
+type: alias
 
+alias to: source.address
 
 --
 

diff --git a/filebeat/module/apache/_meta/fields.yml b/filebeat/module/apache/_meta/fields.yml
@@ -14,7 +14,7 @@
           fields:
             - name: remote_ip
               type: alias
-              path: apache.access.remote_ip
+              path: source.address
               migration: true
             - name: ssl.protocol
               type: alias

diff --git a/filebeat/module/apache/access/_meta/fields.yml b/filebeat/module/apache/access/_meta/fields.yml
@@ -3,11 +3,6 @@
   description: >
     Contains fields for the Apache HTTP Server access logs.
   fields:
-    - name: remote_ip
-      type: keyword
-      description: >
-        Client IP address or hostname.
-
     - name: ssl.protocol
       type: keyword
       description: >

diff --git a/filebeat/module/apache/access/ingest/default.json b/filebeat/module/apache/access/ingest/default.json
@@ -4,9 +4,9 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "%{IPORHOST:apache.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:apache.access.body_sent.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:apache.access.agent}\")?",
-        "%{IPORHOST:apache.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -",
-        "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:apache.access.remote_ip} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:apache.access.body_sent.bytes}"
+        "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:apache.access.body_sent.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:apache.access.agent}\")?",
+        "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -",
+        "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:apache.access.body_sent.bytes}"
         ],
       "ignore_missing": true
     }
@@ -16,7 +16,7 @@
     }
   }, {
     "grok": {
-      "field": "apache.access.remote_ip",
+      "field": "source.address",
       "ignore_missing": true,
       "patterns": [
         "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"

diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json
@@ -2,7 +2,6 @@
     {
         "@timestamp": "2018-08-10T07:45:56.000Z",
         "apache.access.body_sent.bytes": "1375",
-        "apache.access.remote_ip": "172.30.0.119",
         "apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
         "apache.access.ssl.protocol": "TLSv1.2",
         "ecs.version": "1.0.0-beta2",
@@ -14,6 +13,7 @@
         "input.type": "log",
         "log.offset": 0,
         "service.type": "apache",
+        "source.address": "172.30.0.119",
         "source.ip": "172.30.0.119",
         "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21"
     }

diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json
@@ -2,7 +2,6 @@
     {
         "@timestamp": "2016-12-26T14:16:29.000Z",
         "apache.access.body_sent.bytes": 209,
-        "apache.access.remote_ip": "::1",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "apache.access",
         "event.module": "apache",
@@ -13,14 +12,14 @@
         "input.type": "log",
         "log.offset": 0,
         "service.type": "apache",
+        "source.address": "::1",
         "source.ip": "::1",
         "url.original": "/favicon.ico",
         "user.name": "-"
     },
     {
         "@timestamp": "2016-12-26T16:22:13.000Z",
         "apache.access.body_sent.bytes": 499,
-        "apache.access.remote_ip": "192.168.33.1",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "apache.access",
         "event.module": "apache",
@@ -32,6 +31,7 @@
         "input.type": "log",
         "log.offset": 73,
         "service.type": "apache",
+        "source.address": "192.168.33.1",
         "source.ip": "192.168.33.1",
         "url.original": "/hello",
         "user.name": "-",
@@ -47,7 +47,6 @@
     },
     {
         "@timestamp": "2016-12-26T14:16:48.000Z",
-        "apache.access.remote_ip": "::1",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "apache.access",
         "event.module": "apache",
@@ -56,13 +55,13 @@
         "input.type": "log",
         "log.offset": 238,
         "service.type": "apache",
+        "source.address": "::1",
         "source.ip": "::1",
         "user.name": "-"
     },
     {
         "@timestamp": "2017-05-29T19:02:48.000Z",
         "apache.access.body_sent.bytes": 612,
-        "apache.access.remote_ip": "172.17.0.1",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "apache.access",
         "event.module": "apache",
@@ -74,6 +73,7 @@
         "input.type": "log",
         "log.offset": 285,
         "service.type": "apache",
+        "source.address": "172.17.0.1",
         "source.ip": "172.17.0.1",
         "url.original": "/stringpatch",
         "user.name": "-",
@@ -89,7 +89,6 @@
     {
         "@timestamp": "2017-05-29T19:02:48.000Z",
         "apache.access.body_sent.bytes": 612,
-        "apache.access.remote_ip": "monitoring-server",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "apache.access",
         "event.module": "apache",
@@ -101,6 +100,7 @@
         "input.type": "log",
         "log.offset": 443,
         "service.type": "apache",
+        "source.address": "monitoring-server",
         "source.domain": "monitoring-server",
         "url.original": "/status",
         "user.name": "-",

diff --git a/filebeat/module/apache/fields.go b/filebeat/module/apache/fields.go
diff --git a/filebeat/module/haproxy/_meta/fields.yml b/filebeat/module/haproxy/_meta/fields.yml
@@ -92,10 +92,9 @@
           type: group
           fields:
           - name: ip
-            description: >
-              IP address of the client which initiated the TCP connection to haproxy.
-
-              If connection is via unix socket, socket path is in this field.
+            type: alias
+            path: source.address
+            migration: true
           - name: port
             type: alias
             path: source.port

diff --git a/filebeat/module/haproxy/fields.go b/filebeat/module/haproxy/fields.go
diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json
@@ -5,13 +5,13 @@
             "grok": {
                 "field": "message",
                 "patterns": [
-                    "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)",
+                    "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)",
 
-                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"",
+                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"",
 
-                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}",
+                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}",
 
-                    "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}"
+                    "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}"
                 ],
                 "ignore_missing": false,
                 "pattern_definitions": {
@@ -41,10 +41,10 @@
         },
         {
             "grok": {
-                "field": "haproxy.client.ip",
-                "ignore_missing": true,
+                "field": "source.address",
+                "ignore_failure": true,
                 "patterns": [
-                    "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
+                    "^%{IP:source.ip}$"
                 ]
             }
         },

diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json
@@ -6,7 +6,6 @@
         "event.dataset": "haproxy.log",
         "event.module": "haproxy",
         "fileset.name": "log",
-        "haproxy.client.ip": "1.2.3.4",
         "haproxy.frontend_name": "main",
         "haproxy.mode": "HTTP",
         "haproxy.source": "1.2.3.4",
@@ -15,6 +14,7 @@
         "process.name": "haproxy",
         "process.pid": 24551,
         "service.type": "haproxy",
+        "source.address": "1.2.3.4",
         "source.geo.continent_name": "North America",
         "source.geo.country_iso_code": "US",
         "source.geo.location.lat": 37.751,