From 99e7c8d8477b0c2450ba9389322a165e944027ec Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 28 May 2020 09:16:25 -0500 Subject: [PATCH] [Filebeat] Preserve case of http.request.method (#18359) (#18788) * Preserve case of http.request.method ECS previously specified normalizing http.request.method to lowercase. This resulted in the loss of information. Affects filesets from the following versions: - apache/access (7.7 - 7.8) - elasticsearch/audit (7.7 - 7.8) - iis/access (7.7 - 7.8) - iis/error (7.7 - 7.8) - nginx/access (7.8) - nginx/ingress_controller (7.8) - aws/elb (7.7 - 7.8) - suricata/eve (7.4 - 7.8) - zeek/http (7.8) Closes #18154 (cherry picked from commit 5490eb43c7988cd4bb9742e0102cb98e5355c4ce) --- CHANGELOG.next.asciidoc | 1 + .../module/apache/access/ingest/pipeline.yml | 3 -- .../test/darwin-2.4.23.log-expected.json | 10 ++--- .../access/test/ssl-request.log-expected.json | 4 +- .../access/test/test-vhost.log-expected.json | 2 +- .../apache/access/test/test.log-expected.json | 8 ++-- .../test/ubuntu-2.2.22.log-expected.json | 18 ++++----- .../elasticsearch/audit/ingest/pipeline.yml | 3 -- .../test/test-audit-docker.log-expected.json | 4 +- .../audit/test/test-audit.log-expected.json | 2 +- .../module/iis/access/ingest/pipeline.yml | 3 -- .../test/test-iis-7.2.log-expected.json | 10 ++--- .../test/test-iis-7.5.log-expected.json | 8 ++-- .../test/test-ipv6zone.log-expected.json | 2 +- .../iis/access/test/test.log-expected.json | 10 ++--- filebeat/module/iis/error/ingest/pipeline.yml | 3 -- .../test/iis_error_url.log-expected.json | 14 +++---- .../iis/error/test/test.log-expected.json | 6 +-- .../access/test/access.log-expected.json | 2 +- .../test/test-with-host.log-expected.json | 2 +- .../nginx/access/test/test.log-expected.json | 2 +- .../test/test.log-expected.json | 2 +- .../module/aws/elb/ingest/pipeline.yml | 4 -- .../application-lb-http.log-expected.json | 20 +++++----- .../aws/elb/test/elb-http.log-expected.json | 10 ++--- .../test/example-alb-http.log-expected.json | 14 +++---- .../elb/test/example-http.log-expected.json | 6 +-- .../elb/test/example-https.log-expected.json | 2 +- .../module/suricata/eve/ingest/pipeline.yml | 8 ++-- .../eve/test/eve-alerts.log-expected.json | 40 +++++++++---------- .../eve/test/eve-small.log-expected.json | 4 +- 31 files changed, 106 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ab00afe41e3..295ce94fa99 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -22,6 +22,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] +- Preserve case of http.request.method. ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, aws/elb, suricata/eve. {issue}18154[18154] {pull}18359[18359] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index ff905bd7245..0a9330b68b0 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" -- lowercase: - field: http.request.method - ignore_missing: true - grok: field: source.address ignore_missing: true diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json index 4bf4ca896d6..9c61a6065af 100644 --- a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json @@ -7,7 +7,7 @@ "event.module": "apache", "event.outcome": "success", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 45, "http.response.status_code": 200, "http.version": "1.1", @@ -27,7 +27,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 209, "http.response.status_code": 404, "http.version": "1.1", @@ -63,7 +63,7 @@ "event.module": "apache", "event.outcome": "success", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 45, "http.response.status_code": 200, "http.version": "1.1", @@ -92,7 +92,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 206, "http.response.status_code": 404, "http.version": "1.1", @@ -121,7 +121,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 201, "http.response.status_code": 404, "http.version": "1.1", diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json index 946a3e22dab..9898d82cef0 100644 --- a/filebeat/module/apache/access/test/ssl-request.log-expected.json +++ b/filebeat/module/apache/access/test/ssl-request.log-expected.json @@ -8,7 +8,7 @@ "event.kind": "event", "event.module": "apache", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1375, "http.version": "1.1", "input.type": "log", @@ -30,7 +30,7 @@ "event.kind": "event", "event.module": "apache", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.version": "1.1", "input.type": "log", "log.offset": 276, diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json index 0a593646626..d61237c3c8d 100644 --- a/filebeat/module/apache/access/test/test-vhost.log-expected.json +++ b/filebeat/module/apache/access/test/test-vhost.log-expected.json @@ -8,7 +8,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index 0c1520846fb..7b15274997a 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -7,7 +7,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 209, "http.response.status_code": 404, "http.version": "1.1", @@ -27,7 +27,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, @@ -71,7 +71,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 612, "http.response.status_code": 404, @@ -99,7 +99,7 @@ "event.module": "apache", "event.outcome": "success", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 612, "http.response.status_code": 200, diff --git a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json index 2fbd7b9ffb6..cdf664d927e 100644 --- a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json @@ -7,7 +7,7 @@ "event.module": "apache", "event.outcome": "success", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 491, "http.response.status_code": 200, @@ -33,7 +33,7 @@ "event.module": "apache", "event.outcome": "success", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 484, "http.response.status_code": 200, @@ -61,7 +61,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://192.168.33.72/", "http.response.body.bytes": 504, "http.response.status_code": 404, @@ -89,7 +89,7 @@ "event.module": "apache", "event.outcome": "success", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 484, "http.response.status_code": 200, @@ -117,7 +117,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 504, "http.response.status_code": 404, @@ -145,7 +145,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 504, "http.response.status_code": 404, @@ -173,7 +173,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 498, "http.response.status_code": 404, @@ -201,7 +201,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, @@ -229,7 +229,7 @@ "event.module": "apache", "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 8ad600ca792..ef48280d543 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -40,9 +40,6 @@ processors: ctx.event.outcome = 'failure'; } -- lowercase: - field: http.request.method - ignore_missing: true - set: field: host.id value: "{{elasticsearch.node.id}}" diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json index 457f930622d..f8127900e70 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json @@ -13,7 +13,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg", - "http.request.method": "get", + "http.request.method": "GET", "input.type": "log", "log.offset": 0, "message": "{\"type\": \"audit\", \"timestamp\":\"2019-06-11T15:03:32,102+0000\", \"node.id\":\"Xaq2BFVcQ1OhyMrjL8gNOg\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"172.17.0.1:40380\", \"url.path\":\"/\", \"request.method\":\"GET\", \"request.id\":\"pkduyMB5Tly6xgmkYbZi-A\"}", @@ -37,7 +37,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg", - "http.request.method": "get", + "http.request.method": "GET", "input.type": "log", "log.offset": 690, "message": "{\"type\": \"audit\", \"timestamp\":\"2019-06-11T15:03:32,778+0000\", \"node.id\":\"Xaq2BFVcQ1OhyMrjL8gNOg\", \"event.type\":\"rest\", \"event.action\":\"authentication_failed\", \"user.name\":\"elastic\", \"origin.type\":\"rest\", \"origin.address\":\"172.17.0.1:40380\", \"url.path\":\"/\", \"request.method\":\"GET\", \"request.id\":\"KPgEINaXSbGNaIobp8OcMw\"}", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json index 4d618682910..bb3e1ce38c2 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json @@ -202,7 +202,7 @@ "fileset.name": "audit", "host.id": "y8fa3M5zSSGo1M_KJRMUXw", "http.request.body.content": "\n{\n \"query\" : {\n \"term\" : { \"user\" : \"kimchy\" }\n }\n}\n", - "http.request.method": "get", + "http.request.method": "GET", "input.type": "log", "log.offset": 2056, "message": "{\"@timestamp\":\"2019-01-27T20:15:10,380\", \"node.name\":\"node-0\", \"node.id\":\"y8fa3M5zSSGo1M_KJRMUXw\", \"event.type\":\"rest\", \"event.action\":\"authentication_success\", \"user.name\":\"elastic-admin\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:58955\", \"realm\":\"default_file\", \"url.path\":\"/_search\", \"request.method\":\"GET\", \"request.body\":\"\\n{\\n \\\"query\\\" : {\\n \\\"term\\\" : { \\\"user\\\" : \\\"kimchy\\\" }\\n }\\n}\\n\", \"request.id\":\"WzL_kb6VSvOhAq0twPvHOQ\"}", diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 4437c090c7a..8344cccac1b 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -110,9 +110,6 @@ processors: field: event.type value: connection if: "ctx?.source?.ip != null && ctx?.destination?.ip != null" -- lowercase: - field: http.request.method - ignore_missing: true - append: field: related.ip value: "{{source.ip}}" diff --git a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json index 990d2a171c1..64ad587bb8b 100644 --- a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json @@ -17,7 +17,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 404, "iis.access.sub_status": 0, "iis.access.win32_status": 64, @@ -58,7 +58,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 404, "iis.access.sub_status": 0, "iis.access.win32_status": 2, @@ -99,7 +99,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 401, "iis.access.sub_status": 0, "iis.access.win32_status": 0, @@ -139,7 +139,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 401, "iis.access.sub_status": 0, "iis.access.win32_status": 0, @@ -179,7 +179,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 404, "iis.access.sub_status": 0, "iis.access.win32_status": 64, diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json index 0c3a2abb1b1..95210536925 100644 --- a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json @@ -17,7 +17,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 404, "iis.access.sub_status": 4, "iis.access.win32_status": 2, @@ -57,7 +57,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 200, "iis.access.sub_status": 0, "iis.access.win32_status": 0, @@ -90,7 +90,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 200, "iis.access.sub_status": 0, "iis.access.win32_status": 0, @@ -123,7 +123,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 200, "iis.access.sub_status": 0, "iis.access.win32_status": 0, diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json index 357380f628e..448779366ce 100644 --- a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json +++ b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json @@ -19,7 +19,7 @@ ], "fileset.name": "access", "http.request.body.bytes": 456, - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 123, "http.response.status_code": 200, "http.version": "1.1", diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index 7ef0cfac036..909bffb0e62 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -17,7 +17,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 200, "iis.access.sub_status": 0, "iis.access.win32_status": 0, @@ -63,7 +63,7 @@ "event.outcome": "success", "fileset.name": "access", "http.request.body.bytes": 456, - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 123, "http.response.status_code": 200, "iis.access.site_name": "W3SVC1", @@ -106,7 +106,7 @@ ], "fileset.name": "access", "http.request.body.bytes": 456, - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 123, "http.response.status_code": 200, "http.version": "1.1", @@ -159,7 +159,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 401, "iis.access.sub_status": 0, "iis.access.win32_status": 0, @@ -200,7 +200,7 @@ "connection" ], "fileset.name": "access", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 404, "iis.access.sub_status": 0, "iis.access.win32_status": 2, diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index 4e43aeac0bc..4611744d3c9 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -71,9 +71,6 @@ processors: field: event.type value: connection if: "ctx?.source?.ip != null && ctx?.destination?.ip != null" -- lowercase: - field: http.request.method - ignore_missing: true - append: field: related.ip value: "{{source.ip}}" diff --git a/filebeat/module/iis/error/test/iis_error_url.log-expected.json b/filebeat/module/iis/error/test/iis_error_url.log-expected.json index 03258176f35..0cb2fb038b4 100644 --- a/filebeat/module/iis/error/test/iis_error_url.log-expected.json +++ b/filebeat/module/iis/error/test/iis_error_url.log-expected.json @@ -53,7 +53,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 400, "http.version": "1.1", "iis.error.reason_phrase": "URL", @@ -90,7 +90,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 403, "http.version": "1.1", "iis.error.reason_phrase": "Forbidden", @@ -127,7 +127,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 400, "http.version": "1.1", "iis.error.reason_phrase": "URL", @@ -164,7 +164,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 404, "http.version": "1.1", "iis.error.reason_phrase": "NotFound", @@ -201,7 +201,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 403, "http.version": "1.1", "iis.error.reason_phrase": "Forbidden", @@ -238,7 +238,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "options", + "http.request.method": "OPTIONS", "http.response.status_code": 404, "http.version": "1.1", "iis.error.reason_phrase": "NotFound", @@ -275,7 +275,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 400, "http.version": "1.1", "iis.error.reason_phrase": "URL", diff --git a/filebeat/module/iis/error/test/test.log-expected.json b/filebeat/module/iis/error/test/test.log-expected.json index 8a78dd9876d..50ec549dd6b 100644 --- a/filebeat/module/iis/error/test/test.log-expected.json +++ b/filebeat/module/iis/error/test/test.log-expected.json @@ -16,7 +16,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 503, "http.version": "1.1", "iis.error.reason_phrase": "ConnLimit", @@ -49,7 +49,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 400, "http.version": "1.1", "iis.error.reason_phrase": "Hostname", @@ -91,7 +91,7 @@ "connection" ], "fileset.name": "error", - "http.request.method": "get", + "http.request.method": "GET", "http.response.status_code": 505, "http.version": "2.0", "iis.error.reason_phrase": "Version_N/S", diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index a121dd67613..6de197fcbc4 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -419,4 +419,4 @@ "user_agent.os.version": "10.12", "user_agent.version": "49.0." } -] \ No newline at end of file +] diff --git a/filebeat/module/nginx/access/test/test-with-host.log-expected.json b/filebeat/module/nginx/access/test/test-with-host.log-expected.json index 38695946ca5..3bf2363bd7e 100644 --- a/filebeat/module/nginx/access/test/test-with-host.log-expected.json +++ b/filebeat/module/nginx/access/test/test-with-host.log-expected.json @@ -317,4 +317,4 @@ "user_agent.os.version": "7", "user_agent.version": "15.0.a2" } -] \ No newline at end of file +] diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 247b7a12e21..7c2d730cc3d 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -303,4 +303,4 @@ "user_agent.os.version": "7", "user_agent.version": "15.0.a2" } -] \ No newline at end of file +] diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 686d0375442..78b1390f963 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -821,4 +821,4 @@ "user_agent.os.version": "10.14", "user_agent.version": "72.0." } -] \ No newline at end of file +] diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index a206ccf314a..fc202d7d14e 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -127,10 +127,6 @@ processors: field: event.outcome value: failure - - lowercase: - field: http.request.method - ignore_missing: true - - set: if: "ctx?.aws?.elb?.trace_id != null" field: tracing.trace.id diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json index 093cc1fc2e7..a3acfcbc002 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json @@ -22,7 +22,7 @@ "event.start": "2019-10-11T15:01:06.657000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 0, "http.response.status_code": 460, @@ -67,7 +67,7 @@ "event.start": "2019-10-11T15:01:40.491000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, @@ -112,7 +112,7 @@ "event.start": "2019-10-11T15:01:12.914000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, @@ -157,7 +157,7 @@ "event.start": "2019-10-11T15:01:25.189000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, @@ -202,7 +202,7 @@ "event.start": "2019-10-11T15:02:18.836000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, @@ -247,7 +247,7 @@ "event.start": "2019-10-11T15:02:31.202000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, @@ -292,7 +292,7 @@ "event.start": "2019-10-11T15:03:39.331000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, @@ -341,7 +341,7 @@ "event.start": "2019-10-11T15:55:09.307000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, "http.response.status_code": 200, @@ -390,7 +390,7 @@ "event.start": "2019-10-11T15:55:11.352000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, "http.response.status_code": 200, @@ -439,7 +439,7 @@ "event.start": "2019-10-11T15:55:11.987000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, "http.response.status_code": 200, diff --git a/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json index f8b0d751e75..d2d1e224f05 100644 --- a/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json @@ -18,7 +18,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://18.194.223.56:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, @@ -58,7 +58,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://18.194.223.56:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, @@ -98,7 +98,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, @@ -138,7 +138,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, @@ -178,7 +178,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json index 1a46cee8d85..d9090e9855e 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json @@ -26,7 +26,7 @@ "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 34, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, "http.response.status_code": 200, @@ -71,7 +71,7 @@ "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "https://www.example.com:443/", "http.response.body.bytes": 57, "http.response.status_code": 200, @@ -117,7 +117,7 @@ "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 5, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "https://10.0.2.105:773/", "http.response.body.bytes": 257, "http.response.status_code": 200, @@ -160,7 +160,7 @@ "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 218, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://10.0.0.30:80/", "http.response.body.bytes": 587, "http.response.status_code": 101, @@ -194,7 +194,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 218, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "https://10.0.0.30:443/", "http.response.body.bytes": 786, "http.response.status_code": 101, @@ -234,7 +234,7 @@ "event.start": "2018-11-30T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 34, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, "http.response.status_code": 200, @@ -272,7 +272,7 @@ "event.start": "2018-11-30T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 34, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, "http.response.status_code": 502, diff --git a/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json index 72f9a57f6e3..d8de28648da 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json @@ -18,7 +18,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 29, "http.response.status_code": 200, @@ -43,7 +43,7 @@ "event.outcome": "failure", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 0, "http.response.status_code": 503, @@ -68,7 +68,7 @@ "event.outcome": "failure", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://www.example.com:80-", "http.response.body.bytes": 0, "http.response.status_code": 400, diff --git a/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json index ef09a37d579..871b30b3153 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json @@ -20,7 +20,7 @@ "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "https://www.example.com:443/", "http.response.body.bytes": 57, "http.response.status_code": 200, diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml index 4da1873e26a..63a79ce71de 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml @@ -2,10 +2,10 @@ description: Pipeline for parsing Suricata EVE logs processors: - - lowercase: - field: suricata.eve.http.http_method - target_field: http.request.method - ignore_missing: true + - set: + value: "{{suricata.eve.http.http_method}}" + field: http.request.method + if: "ctx?.suricata?.eve?.http?.http_method != null" - rename: field: suricata.eve.http.status target_field: http.response.status_code diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index e7c96246e7c..793ce164746 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -30,7 +30,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1121, "http.response.status_code": 200, "input.type": "log", @@ -107,7 +107,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1121, "http.response.status_code": 200, "input.type": "log", @@ -184,7 +184,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1126, "http.response.status_code": 200, "input.type": "log", @@ -261,7 +261,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1121, "http.response.status_code": 200, "input.type": "log", @@ -338,7 +338,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1121, "http.response.status_code": 200, "input.type": "log", @@ -415,7 +415,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1126, "http.response.status_code": 200, "input.type": "log", @@ -492,7 +492,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1138, "http.response.status_code": 200, "input.type": "log", @@ -569,7 +569,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 0, "http.response.status_code": 304, "input.type": "log", @@ -646,7 +646,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2601, "http.response.status_code": 200, "input.type": "log", @@ -723,7 +723,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1241, "http.response.status_code": 200, "input.type": "log", @@ -800,7 +800,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2687, "http.response.status_code": 200, "input.type": "log", @@ -877,7 +877,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2688, "http.response.status_code": 200, "input.type": "log", @@ -954,7 +954,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2601, "http.response.status_code": 200, "input.type": "log", @@ -1031,7 +1031,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2687, "http.response.status_code": 200, "input.type": "log", @@ -1108,7 +1108,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2688, "http.response.status_code": 200, "input.type": "log", @@ -1185,7 +1185,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2687, "http.response.status_code": 200, "input.type": "log", @@ -1262,7 +1262,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2691, "http.response.status_code": 200, "input.type": "log", @@ -1339,7 +1339,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 2687, "http.response.status_code": 200, "input.type": "log", @@ -1416,7 +1416,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 0, "input.type": "log", "log.offset": 14767, @@ -1492,7 +1492,7 @@ "allowed" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 0, "input.type": "log", "log.offset": 15651, diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 2f53173a641..ec02bba8dd1 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -119,7 +119,7 @@ "protocol" ], "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1155, "http.response.status_code": 200, "input.type": "log", @@ -171,7 +171,7 @@ "file.path": "/ssdp/device-desc.xml", "file.size": 1071, "fileset.name": "eve", - "http.request.method": "get", + "http.request.method": "GET", "http.response.body.bytes": 1071, "http.response.status_code": 200, "input.type": "log",