From 0ccffa01dacba423919c017c0d70797a7a818dd2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 15 May 2020 14:18:41 +0300 Subject: [PATCH] Fix source.address not being set for nginx ingress_controller Signed-off-by: chrismark Co-authored-by: chendo (cherry picked from commit 8e29ab7) --- CHANGELOG.next.asciidoc | 1 + .../ingress_controller/ingest/pipeline.json | 2 +- .../test/test.log-expected.json | 136 ++++++++++++++---- 3 files changed, 114 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 51d23bac8ef..e51348d9e36 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -128,6 +128,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed typo in log message. {pull}17897[17897] - Unescape file name from SQS message. {pull}18370[18370] - Improve cisco asa and ftd pipelines' failure handler to avoid mapping temporary fields. {issue}18391[18391] {pull}18392[18392] +- Fix source.address not being set for nginx ingress_controller {pull}18511[18511] - Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472] - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534] diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.json b/filebeat/module/nginx/ingress_controller/ingest/pipeline.json index e660f22f022..69facbcc94f 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.json +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.json @@ -54,7 +54,7 @@ }, { "script": { - "if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0", + "if": "ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0", "lang": "painless", "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } try { ctx.source.address = null; if (ctx.nginx.ingress_controller.remote_ip_list == null) { return; } def found = false; for (def item : ctx.nginx.ingress_controller.remote_ip_list) { if (!isPrivate(params.dot, item)) { ctx.source.address = item; found = true; break; } } if (!found) { ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0]; }} catch (Exception e) { ctx.source.address = null; }", "params": { diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 2dc9d1afbce..98a5832378f 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -25,8 +25,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products", "user.name": "-", "user_agent.device.name": "Other", @@ -60,8 +64,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", @@ -95,8 +103,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", @@ -130,8 +142,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", @@ -160,13 +176,17 @@ ], "nginx.ingress_controller.upstream.alternative_name": "", "nginx.ingress_controller.upstream.name": "", + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "-" + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", }, { "@timestamp": "2020-02-07T11:50:09.000Z", @@ -189,13 +209,17 @@ ], "nginx.ingress_controller.upstream.alternative_name": "", "nginx.ingress_controller.upstream.name": "", + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", - "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "-" + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", + "url.original": "/products/42" }, { "@timestamp": "2020-02-07T11:55:05.000Z", @@ -223,8 +247,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", @@ -258,8 +286,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", @@ -296,8 +328,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", @@ -334,8 +370,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2", "user.name": "-", "user_agent.device.name": "Other", @@ -372,8 +412,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", @@ -410,8 +454,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", @@ -448,8 +496,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", @@ -486,8 +538,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42", "user.name": "-", "user_agent.device.name": "Other", @@ -524,8 +580,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/", "user.name": "-", "user_agent.device.name": "Other", @@ -562,8 +622,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", @@ -600,8 +664,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.002, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2", "user.name": "-", "user_agent.device.name": "Other", @@ -638,8 +706,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", @@ -676,8 +748,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/products/42?address=delhi+technological+university", "user.name": "-", "user_agent.device.name": "Other", @@ -711,8 +787,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.001, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2", "user.name": "-", "user_agent.device.name": "Other", @@ -749,8 +829,12 @@ "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", @@ -787,8 +871,12 @@ "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.time": 0.0, + "related.ip": [ + "192.168.64.1" + ], "service.type": "nginx", - "source.address": "", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", "url.original": "/v2/some", "user.name": "-", "user_agent.device.name": "Other", @@ -799,4 +887,4 @@ "user_agent.os.version": "10.14", "user_agent.version": "72.0." } -] \ No newline at end of file +]