Fix source.address not being set for nginx ingress_controller

Signed-off-by: chrismark <[email protected]> Co-authored-by: chendo <[email protected]> (cherry picked from commit 8e29ab7)
elastic · May 15, 2020 · 0ccffa0 · 0ccffa0
1 parent f39ac65
commit 0ccffa0
Show file tree

Hide file tree

Showing 3 changed files with 114 additions and 25 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -128,6 +128,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed typo in log message. {pull}17897[17897]
 - Unescape file name from SQS message. {pull}18370[18370]
 - Improve cisco asa and ftd pipelines' failure handler to avoid mapping temporary fields. {issue}18391[18391] {pull}18392[18392]
+- Fix source.address not being set for nginx ingress_controller {pull}18511[18511]
 - Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472]
 - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534]
 

diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.json b/filebeat/module/nginx/ingress_controller/ingest/pipeline.json
@@ -54,7 +54,7 @@
         },
         {
             "script": {
-                "if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0",
+                "if": "ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0",
                 "lang": "painless",
                 "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken());       int secondByte = Integer.parseInt(tok.nextToken());       if (firstByte == 10) {         return true;       }       if (firstByte == 192 && secondByte == 168) {         return true;       }       if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {         return true;       }       if (firstByte == 127) {         return true;       }       return false;     } catch (Exception e) {       return false;     }   }   try {    ctx.source.address = null;    if (ctx.nginx.ingress_controller.remote_ip_list == null) { return; }    def found = false;    for (def item : ctx.nginx.ingress_controller.remote_ip_list) {        if (!isPrivate(params.dot, item)) {            ctx.source.address = item;            found = true;            break;        }    }    if (!found) {     ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0];   }} catch (Exception e) { ctx.source.address = null; }",
                 "params": {

diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json
@@ -25,8 +25,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -60,8 +64,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -95,8 +103,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -130,8 +142,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -160,13 +176,17 @@
         ],
         "nginx.ingress_controller.upstream.alternative_name": "",
         "nginx.ingress_controller.upstream.name": "",
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "-"
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
     },
     {
         "@timestamp": "2020-02-07T11:50:09.000Z",
@@ -189,13 +209,17 @@
         ],
         "nginx.ingress_controller.upstream.alternative_name": "",
         "nginx.ingress_controller.upstream.name": "",
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
-        "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "-"
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
+        "url.original": "/products/42"
     },
     {
         "@timestamp": "2020-02-07T11:55:05.000Z",
@@ -223,8 +247,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -258,8 +286,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -296,8 +328,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -334,8 +370,12 @@
         "nginx.ingress_controller.upstream.response.length": 61,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -372,8 +412,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -410,8 +454,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -448,8 +496,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -486,8 +538,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -524,8 +580,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -562,8 +622,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -600,8 +664,12 @@
         "nginx.ingress_controller.upstream.response.length": 61,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.002,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -638,8 +706,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -676,8 +748,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/products/42?address=delhi+technological+university",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -711,8 +787,12 @@
         "nginx.ingress_controller.upstream.response.length": 61,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.001,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -749,8 +829,12 @@
         "nginx.ingress_controller.upstream.response.length": 59,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/favicon.ico",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -787,8 +871,12 @@
         "nginx.ingress_controller.upstream.response.length": 61,
         "nginx.ingress_controller.upstream.response.status_code": 200,
         "nginx.ingress_controller.upstream.response.time": 0.0,
+        "related.ip": [
+            "192.168.64.1"
+        ],
         "service.type": "nginx",
-        "source.address": "",
+        "source.address": "192.168.64.1",
+        "source.ip": "192.168.64.1",
         "url.original": "/v2/some",
         "user.name": "-",
         "user_agent.device.name": "Other",
@@ -799,4 +887,4 @@
         "user_agent.os.version": "10.14",
         "user_agent.version": "72.0."
     }
-]
+]