[Filebeat] Add support to unix sockets in Nginx module (#10944)

elastic · Apr 15, 2019 · 9b7c782 · 9b7c782
1 parent 5e22c55
commit 9b7c782
Show file tree

Hide file tree

Showing 6 changed files with 87 additions and 18 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -11453,15 +11453,6 @@ alias to: http.response.body.bytes
 
 --
 
-*`nginx.access.remote_ip`*::
-+
---
-type: alias
-
-alias to: source.address
-
---
-
 *`nginx.access.user_name`*::
 +
 --

diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml
@@ -14,10 +14,6 @@
       type: alias
       path: http.response.body.bytes
       migration: true
-    - name: remote_ip
-      type: alias
-      path: source.address
-      migration: true
     - name: user_name
       type: alias
       path: user.name

diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json
@@ -5,7 +5,7 @@
             "grok": {
                 "field": "message",
                 "patterns": [
-                    "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:user_agent.original}\""
+                    "\"?(?:%{IP_LIST:nginx.access.remote_ip_list}|%{DATA:source.address}) - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:user_agent.original}\""
                 ],
                 "pattern_definitions": {
                     "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*"
@@ -31,7 +31,15 @@
         {
             "split": {
                 "field": "nginx.access.remote_ip_list",
-                "separator": "\"?,?\\s+"
+                "separator": "\"?,?\\s+",
+                "ignore_missing": true
+            }
+        },
+        {
+            "split": {
+                "field": "nginx.access.origin",
+                "separator": "\"?,?\\s+",
+                "ignore_missing": true
             }
         },
         {
@@ -43,17 +51,24 @@
         {
             "script": {
                 "lang": "painless",
-                "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken());       int secondByte = Integer.parseInt(tok.nextToken());       if (firstByte == 10) {         return true;       }       if (firstByte == 192 && secondByte == 168) {         return true;       }       if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {         return true;       }       if (firstByte == 127) {         return true;       }       return false;     } catch (Exception e) {       return false;     }   }   def found = false;   for (def item : ctx.nginx.access.remote_ip_list) {     if (!isPrivate(params.dot, item)) {       ctx.source.ip = item;       found = true;       break;     }   }   if (!found) {     ctx.source.ip = ctx.nginx.access.remote_ip_list[0];   }",
+                "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken());       int secondByte = Integer.parseInt(tok.nextToken());       if (firstByte == 10) {         return true;       }       if (firstByte == 192 && secondByte == 168) {         return true;       }       if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {         return true;       }       if (firstByte == 127) {         return true;       }       return false;     } catch (Exception e) {       return false;     }   }   try {    ctx.source.ip = null;    if (ctx.nginx.access.remote_ip_list == null) { return; }    def found = false;    for (def item : ctx.nginx.access.remote_ip_list) {        if (!isPrivate(params.dot, item)) {            ctx.source.ip = item;            found = true;            break;        }    }    if (!found) {     ctx.source.ip = ctx.nginx.access.remote_ip_list[0];   }} catch (Exception e) { ctx.source.ip = null; }",
                 "params": {
                     "dot": "."
                 }
             }
         },
+        {
+            "remove": {
+                "field": "source.ip",
+                "if": "ctx.source.ip == null"
+            }
+        },
         {
             "convert": {
                 "field": "source.ip",
                 "target_field": "source.address",
-                "type": "string"
+                "type": "string",
+                "ignore_missing": true
             }
         },
         {

diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log
@@ -5,3 +5,6 @@
 "10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront"
 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)"
 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-"
+unix: - - [26/Feb/2019:15:39:42 +0100] "hello" 400 173 "-" "-"
+localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
+localhost, localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -219,5 +219,69 @@
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "-"
+    },
+    {
+        "@timestamp": "2019-02-26T14:39:42.000Z",
+        "ecs.version": "1.0.0",
+        "event.dataset": "nginx.access",
+        "event.module": "nginx",
+        "fileset.name": "access",
+        "http.request.referrer": "-",
+        "http.response.body.bytes": 173,
+        "http.response.status_code": 400,
+        "input.type": "log",
+        "log.offset": 1184,
+        "service.type": "nginx",
+        "source.address": "unix:",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Other",
+        "user_agent.original": "-"
+    },
+    {
+        "@timestamp": "2017-05-29T19:02:48.000Z",
+        "ecs.version": "1.0.0",
+        "event.dataset": "nginx.access",
+        "event.module": "nginx",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.body.bytes": 612,
+        "http.response.status_code": 200,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 1247,
+        "service.type": "nginx",
+        "source.address": "localhost",
+        "url.original": "/test2",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Firefox Alpha",
+        "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2",
+        "user_agent.os.name": "Windows 7",
+        "user_agent.version": "15.0.a2"
+    },
+    {
+        "@timestamp": "2017-05-29T19:02:48.000Z",
+        "ecs.version": "1.0.0",
+        "event.dataset": "nginx.access",
+        "event.module": "nginx",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.body.bytes": 612,
+        "http.response.status_code": 200,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 1398,
+        "service.type": "nginx",
+        "source.address": "localhost, localhost",
+        "url.original": "/test2",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Firefox Alpha",
+        "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2",
+        "user_agent.os.name": "Windows 7",
+        "user_agent.version": "15.0.a2"
     }
 ]
diff --git a/filebeat/module/nginx/fields.go b/filebeat/module/nginx/fields.go