diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 984929851bf..2a75bddfdbd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -111,6 +111,8 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Cover empty request data, url and version in Apache2 module{pull}10846[10846] - Fix a bug when converting NetFlow fields to snake_case. {pull}10950[10950] - Fix a bug with the convert_timezone option using the incorrect timezone field. {issue}11055[11055] {pull}11164[11164] +- Change URLPATH grok pattern to support brackets. {issue}11135[11135] {pull}11252[11252] +- Add support for iis log with different address format. {issue}11255[11255] {pull}11256[11256] *Heartbeat* diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json index 1c8d2043375..eea08162950 100644 --- a/filebeat/module/iis/access/ingest/default.json +++ b/filebeat/module/iis/access/ingest/default.json @@ -4,11 +4,15 @@ "grok": { "field": "message", "patterns":[ - "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:iis.access.server_ip} %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.referrer} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.request_time_ms}", + "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:iis.access.server_ip} %{WORD:iis.access.method} %{URIPATHWITHBRACKET:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.referrer} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.request_time_ms}", "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:iis.access.referrer} %{NOTSPACE:iis.access.hostname} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.body_sent.bytes} %{NUMBER:iis.access.body_received.bytes} %{NUMBER:iis.access.request_time_ms}", "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:iis.access.server_ip} %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} HTTP/%{NUMBER:iis.access.http_version} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:iis.access.referrer} %{NOTSPACE:iis.access.hostname} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.body_sent.bytes} %{NUMBER:iis.access.body_received.bytes} %{NUMBER:iis.access.request_time_ms}", - "%{TIMESTAMP_ISO8601:iis.access.time} \\[%{IPORHOST:iis.access.server_ip}\\]\\(http://%{IPORHOST:iis.access.server_ip}\\) %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} \\[%{IPORHOST:iis.access.remote_ip}\\]\\(http://%{IPORHOST:iis.access.remote_ip}\\) %{NOTSPACE:iis.access.agent} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.request_time_ms}" - ], + "%{TIMESTAMP_ISO8601:iis.access.time} \\[%{IPORHOST:iis.access.server_ip}\\]\\(http://%{IPORHOST:iis.access.server_ip}\\) %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} \\[%{IPORHOST:iis.access.remote_ip}\\]\\(http://%{IPORHOST:iis.access.remote_ip}\\) %{NOTSPACE:iis.access.agent} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.request_time_ms}", + "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:iis.access.server_ip} %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} %{NOTSPACE:iis.access.agent} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.request_time_ms}" + ], + "pattern_definitions": { + "URIPATHWITHBRACKET": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\\-\\[\\]]*)+" + }, "ignore_missing": true } }, { diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log b/filebeat/module/iis/access/test/test-iis-7.5.log index 750ddb8b5dd..115756a78ce 100644 --- a/filebeat/module/iis/access/test/test-iis-7.5.log +++ b/filebeat/module/iis/access/test/test-iis-7.5.log @@ -3,3 +3,5 @@ #Date: 2018-08-28 18:24:25 #Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) sc-status sc-substatus sc-win32-status time-taken 2018-08-28 18:24:25 [10.100.220.70](http://10.100.220.70) GET / - 80 - [10.100.118.31](http://10.100.118.31) Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729) 404 4 2 792 +2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15 +2019-03-06 18:43:17 2001:cdba:0000:0000:0000:0000:3257:9652 GET /health-monitoring - 80 - 2001:cdba:0000:0000:0000:0000:3257:9652 - 200 0 0 15 diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json index 7dcbfc80515..ff34cb75e92 100644 --- a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json @@ -23,5 +23,55 @@ "input.type": "log", "offset": 244, "prospector.type": "log" + }, + { + "@timestamp": "2019-03-06T18:43:17.000Z", + "event.dataset": "iis.access", + "fileset.module": "iis", + "fileset.name": "access", + "iis.access.method": "GET", + "iis.access.port": "80", + "iis.access.query_string": "-", + "iis.access.remote_ip": "10.0.140.2", + "iis.access.request_time_ms": "15", + "iis.access.response_code": "200", + "iis.access.server_ip": "10.0.140.107", + "iis.access.sub_status": "0", + "iis.access.url": "/health-monitoring", + "iis.access.user_agent.device": "Other", + "iis.access.user_agent.name": "Other", + "iis.access.user_agent.original": "-", + "iis.access.user_agent.os": "Other", + "iis.access.user_agent.os_name": "Other", + "iis.access.user_name": "-", + "iis.access.win32_status": "0", + "input.type": "log", + "offset": 532, + "prospector.type": "log" + }, + { + "@timestamp": "2019-03-06T18:43:17.000Z", + "event.dataset": "iis.access", + "fileset.module": "iis", + "fileset.name": "access", + "iis.access.method": "GET", + "iis.access.port": "80", + "iis.access.query_string": "-", + "iis.access.remote_ip": "2001:cdba:0000:0000:0000:0000:3257:9652", + "iis.access.request_time_ms": "15", + "iis.access.response_code": "200", + "iis.access.server_ip": "2001:cdba:0000:0000:0000:0000:3257:9652", + "iis.access.sub_status": "0", + "iis.access.url": "/health-monitoring", + "iis.access.user_agent.device": "Other", + "iis.access.user_agent.name": "Other", + "iis.access.user_agent.original": "-", + "iis.access.user_agent.os": "Other", + "iis.access.user_agent.os_name": "Other", + "iis.access.user_name": "-", + "iis.access.win32_status": "0", + "input.type": "log", + "offset": 619, + "prospector.type": "log" } ] \ No newline at end of file diff --git a/filebeat/module/iis/access/test/test.log b/filebeat/module/iis/access/test/test.log index a0c219151d3..c054eaf6557 100644 --- a/filebeat/module/iis/access/test/test.log +++ b/filebeat/module/iis/access/test/test.log @@ -12,4 +12,6 @@ #Version: 1.0 #Date: 2018-01-01 10:11:12 #Fields: date time s-sitename s-computername s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken -2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - - example.com 200 0 0 123 456 789 +2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789 +2018-12-31 12:52:33 10.44.0.136 GET / redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()} 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0 +2018-12-31 12:52:33 10.44.0.136 GET /${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 0 diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index d0b2fef2cec..f900544fb09 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -96,16 +96,71 @@ "iis.access.sub_status": "0", "iis.access.url": "/", "iis.access.user_agent.device": "Other", - "iis.access.user_agent.major": "57", + "iis.access.user_agent.major": "70", "iis.access.user_agent.minor": "0", - "iis.access.user_agent.name": "Firefox", - "iis.access.user_agent.original": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", + "iis.access.user_agent.name": "Chrome", + "iis.access.user_agent.original": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36", + "iis.access.user_agent.os": "Mac OS X 10.14.0", + "iis.access.user_agent.os_major": "10", + "iis.access.user_agent.os_minor": "14", + "iis.access.user_agent.os_name": "Mac OS X", + "iis.access.user_agent.patch": "3538", + "iis.access.user_name": "-", + "iis.access.win32_status": "0", + "input.type": "log", + "offset": 1204, + "prospector.type": "log" + }, + { + "@timestamp": "2018-12-31T12:52:33.000Z", + "event.dataset": "iis.access", + "fileset.module": "iis", + "fileset.name": "access", + "iis.access.method": "GET", + "iis.access.port": "443", + "iis.access.query_string": "redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()}", + "iis.access.referrer": "-", + "iis.access.remote_ip": "10.50.6.188", + "iis.access.request_time_ms": "0", + "iis.access.response_code": "401", + "iis.access.server_ip": "10.44.0.136", + "iis.access.sub_status": "0", + "iis.access.url": "/", + "iis.access.user_agent.device": "Other", + "iis.access.user_agent.name": "Other", + "iis.access.user_agent.original": "Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0)", "iis.access.user_agent.os": "Windows", "iis.access.user_agent.os_name": "Windows", "iis.access.user_name": "-", "iis.access.win32_status": "0", "input.type": "log", - "offset": 1204, + "offset": 1447, + "prospector.type": "log" + }, + { + "@timestamp": "2018-12-31T12:52:33.000Z", + "event.dataset": "iis.access", + "fileset.module": "iis", + "fileset.name": "access", + "iis.access.method": "GET", + "iis.access.port": "443", + "iis.access.query_string": "-", + "iis.access.referrer": "-", + "iis.access.remote_ip": "10.50.6.188", + "iis.access.request_time_ms": "0", + "iis.access.response_code": "404", + "iis.access.server_ip": "10.44.0.136", + "iis.access.sub_status": "0", + "iis.access.url": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", + "iis.access.user_agent.device": "Other", + "iis.access.user_agent.name": "Other", + "iis.access.user_agent.original": "Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0)", + "iis.access.user_agent.os": "Windows", + "iis.access.user_agent.os_name": "Windows", + "iis.access.user_name": "-", + "iis.access.win32_status": "2", + "input.type": "log", + "offset": 1802, "prospector.type": "log" } ] \ No newline at end of file