From a10c1b768bbb56bda8742982273aa08c0e9b63ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 29 May 2017 21:08:21 +0200 Subject: [PATCH] Allow string characters in browser patch version Both for NGINX and Apache logs --- CHANGELOG.asciidoc | 1 + filebeat/docs/fields.asciidoc | 4 +- .../module/apache2/access/_meta/fields.yml | 2 +- filebeat/module/apache2/access/test/test.log | 1 + .../access/test/test.log-expected.json | 52 ++++++++++++++++++- filebeat/module/nginx/access/_meta/fields.yml | 2 +- filebeat/module/nginx/access/test/test.log | 1 + .../nginx/access/test/test.log-expected.json | 51 ++++++++++++++++++ 8 files changed, 109 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d8c7cf79cfa6..d33f9c738d4c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -37,6 +37,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Filebeat* - Fix race condition on harvester stopping with reloading enabled. {issue}3779[3779] - Fix recursive glob config parsing and resolution across restarts. {pull}4269[4269] +- Allow string characters in user agent patch version (NGINX and Apache) {pull}4415[4415] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index a6d408f5f1da..bbe7e419a646 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -155,7 +155,7 @@ The minor version of the user agent. [float] === apache2.access.user_agent.patch -type: long +type: keyword The patch version of the user agent. @@ -1063,7 +1063,7 @@ The minor version of the user agent. [float] === nginx.access.user_agent.patch -type: long +type: keyword The patch version of the user agent. diff --git a/filebeat/module/apache2/access/_meta/fields.yml b/filebeat/module/apache2/access/_meta/fields.yml index c245407dbb47..97fabdc5cab3 100644 --- a/filebeat/module/apache2/access/_meta/fields.yml +++ b/filebeat/module/apache2/access/_meta/fields.yml @@ -61,7 +61,7 @@ description: > The minor version of the user agent. - name: patch - type: long + type: keyword description: > The patch version of the user agent. - name: name diff --git a/filebeat/module/apache2/access/test/test.log b/filebeat/module/apache2/access/test/test.log index a19899c8ce66..c04f25846baf 100644 --- a/filebeat/module/apache2/access/test/test.log +++ b/filebeat/module/apache2/access/test/test.log @@ -1,3 +1,4 @@ ::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209 192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" ::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 - +172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json index 7a7e78f4633e..dc2c9a8883d0 100644 --- a/filebeat/module/apache2/access/test/test.log-expected.json +++ b/filebeat/module/apache2/access/test/test.log-expected.json @@ -138,6 +138,56 @@ "sort": [ 1482761808000 ] + }, + { + "_id": "AVxVs7QZsqw9BQCgtCgc", + "_index": "filebeat-6.0.0-alpha2-2017.05.29", + "_score": null, + "_source": { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache2": { + "access": { + "body_sent": { + "bytes": "612" + }, + "http_version": "1.1", + "method": "GET", + "referrer": "-", + "remote_ip": "172.17.0.1", + "response_code": "404", + "url": "/stringpatch", + "user_agent": { + "device": "Other", + "major": "15", + "minor": "0", + "name": "Firefox Alpha", + "os": "Windows 7", + "os_name": "Windows 7", + "patch": "a2" + }, + "user_name": "-" + } + }, + "beat": { + "hostname": "X1", + "name": "X1", + "version": "6.0.0-alpha2" + }, + "offset": 443, + "prospector": { + "type": "log" + }, + "read_timestamp": "2017-05-29T19:34:14.378Z", + "source": "/home/exekias/go/src/github.com/elastic/beats/filebeat/apache2.log" + }, + "_type": "doc", + "fields": { + "@timestamp": [ + 1496084568000 + ] + }, + "sort": [ + 1496084568000 + ] } ] - diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml index ca8e92824613..0b5f1eb275ed 100644 --- a/filebeat/module/nginx/access/_meta/fields.yml +++ b/filebeat/module/nginx/access/_meta/fields.yml @@ -61,7 +61,7 @@ description: > The minor version of the user agent. - name: patch - type: long + type: keyword description: > The patch version of the user agent. - name: name diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log index 5b5e1a464f67..952c54c202f8 100644 --- a/filebeat/module/nginx/access/test/test.log +++ b/filebeat/module/nginx/access/test/test.log @@ -1 +1,2 @@ 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 957394db25d8..afb3999cefd8 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -54,5 +54,56 @@ "sort": [ 1481105107000 ] + }, + { + "_id": "AVxVuTJrsqw9BQCgtCgi", + "_index": "filebeat-6.0.0-alpha2-2017.05.29", + "_score": null, + "_source": { + "@timestamp": "2017-05-29T19:02:48.000Z", + "beat": { + "hostname": "X1", + "name": "X1", + "version": "6.0.0-alpha2" + }, + "nginx": { + "access": { + "body_sent": { + "bytes": "612" + }, + "http_version": "1.1", + "method": "GET", + "referrer": "-", + "remote_ip": "172.17.0.1", + "response_code": "404", + "url": "/stringpatch", + "user_agent": { + "device": "Other", + "major": "15", + "minor": "0", + "name": "Firefox Alpha", + "os": "Windows 7", + "os_name": "Windows 7", + "patch": "a2" + }, + "user_name": "-" + } + }, + "offset": 341, + "prospector": { + "type": "log" + }, + "read_timestamp": "2017-05-29T19:40:14.373Z", + "source": "/home/exekias/go/src/github.com/elastic/beats/filebeat/nginx.log" + }, + "_type": "doc", + "fields": { + "@timestamp": [ + 1496084568000 + ] + }, + "sort": [ + 1496084568000 + ] } ]