-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Parameterizing Painless script literals (#9770)
* Parameterizing some script literals * Fixing script property name * Fixing syntax error * Reverting nginx change * Reverting auditd pipeline change (for now) * Work around painless limitation * Working around Painless issue for nginx module ingest pipeline * Parameterizing one more script in redis module ingest pipeline * Adding back lang field to explicitly set script language to Painless * Reformatting nginx access log * Fixing indent
- Loading branch information
1 parent
3559e58
commit 2c6030d
Showing
3 changed files
with
222 additions
and
160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,112 +1,139 @@ | ||
| { | ||
| "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", | ||
| "processors": [{ | ||
| "grok": { | ||
| "field": "message", | ||
| "patterns":[ | ||
| "\"?%{IP_LIST:network.forwarded_ip} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:nginx.access.body_sent.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:nginx.access.agent}\"" | ||
| ], | ||
| "pattern_definitions": { | ||
| "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*" | ||
| }, | ||
| "ignore_missing": true | ||
| } | ||
| }, { | ||
| "grok": { | ||
| "field": "nginx.access.info", | ||
| "patterns": [ | ||
| "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}", | ||
| "" | ||
| ], | ||
| "ignore_missing": true | ||
| } | ||
| }, { | ||
| "remove": { | ||
| "field": "nginx.access.info" | ||
| } | ||
| }, { | ||
| "split": { | ||
| "field": "network.forwarded_ip", | ||
| "separator": "\"?,?\\s+" | ||
| } | ||
| }, { | ||
| "set": { | ||
| "field": "source.ip", | ||
| "value": "" | ||
| } | ||
| }, { | ||
| "script": { | ||
| "lang": "painless", | ||
| "inline": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, '.'); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.network.forwarded_ip) { if (!isPrivate(item)) { ctx.source.ip = item; found = true; break; } } if (!found) { ctx.source.ip = ctx.network.forwarded_ip[0]; }" | ||
| } | ||
| }, { | ||
| "remove":{ | ||
| "field": "message" | ||
| } | ||
| }, { | ||
| "rename": { | ||
| "field": "@timestamp", | ||
| "target_field": "read_timestamp" | ||
| } | ||
| }, { | ||
| "date": { | ||
| "field": "nginx.access.time", | ||
| "target_field": "@timestamp", | ||
| "formats": ["dd/MMM/YYYY:H:m:s Z"] | ||
| } | ||
| }, { | ||
| "remove": { | ||
| "field": "nginx.access.time" | ||
| } | ||
|
|
||
| }, { "user_agent": { "field": "nginx.access.agent" } | ||
| }, { | ||
| "rename": { | ||
| "field": "nginx.access.agent", | ||
| "target_field": "user_agent.original" | ||
| } | ||
| }, { | ||
| "rename": { | ||
| "field": "user_agent.os", | ||
| "target_field": "user_agent.os.full_name", | ||
| "ignore_missing": true | ||
| } | ||
| }, { | ||
| "rename": { | ||
| "field": "user_agent.os_name", | ||
| "target_field": "user_agent.os.name", | ||
| "ignore_missing": true | ||
| } | ||
| }, { | ||
| "rename": { | ||
| "field": "user_agent.os_major", | ||
| "target_field": "user_agent.os.major", | ||
| "ignore_missing": true | ||
| } | ||
| }, { | ||
| "rename": { | ||
| "field": "user_agent.os_minor", | ||
| "target_field": "user_agent.os.minor", | ||
| "ignore_missing": true | ||
| } | ||
| }, { | ||
| "rename": { | ||
| "field": "user_agent.os_patch", | ||
| "target_field": "user_agent.os.patch", | ||
| "ignore_missing": true | ||
| } | ||
|
|
||
| },{ | ||
| "geoip": { | ||
| "field": "source.ip", | ||
| "target_field": "source.geo", | ||
| "ignore_missing": true | ||
| } | ||
| }], | ||
| "on_failure" : [{ | ||
| "set" : { | ||
| "field" : "error.message", | ||
| "value" : "{{ _ingest.on_failure_message }}" | ||
| } | ||
| }] | ||
| "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", | ||
| "processors": [ | ||
| { | ||
| "grok": { | ||
| "field": "message", | ||
| "patterns": [ | ||
| "\"?%{IP_LIST:network.forwarded_ip} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:nginx.access.body_sent.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:nginx.access.agent}\"" | ||
| ], | ||
| "pattern_definitions": { | ||
| "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*" | ||
| }, | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "grok": { | ||
| "field": "nginx.access.info", | ||
| "patterns": [ | ||
| "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}", | ||
| "" | ||
| ], | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "remove": { | ||
| "field": "nginx.access.info" | ||
| } | ||
| }, | ||
| { | ||
| "split": { | ||
| "field": "network.forwarded_ip", | ||
| "separator": "\"?,?\\s+" | ||
| } | ||
| }, | ||
| { | ||
| "set": { | ||
| "field": "source.ip", | ||
| "value": "" | ||
| } | ||
| }, | ||
| { | ||
| "script": { | ||
| "lang": "painless", | ||
| "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.network.forwarded_ip) { if (!isPrivate(params.dot, item)) { ctx.source.ip = item; found = true; break; } } if (!found) { ctx.source.ip = ctx.network.forwarded_ip[0]; }", | ||
| "params": { | ||
| "dot": "." | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "remove": { | ||
| "field": "message" | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "@timestamp", | ||
| "target_field": "read_timestamp" | ||
| } | ||
| }, | ||
| { | ||
| "date": { | ||
| "field": "nginx.access.time", | ||
| "target_field": "@timestamp", | ||
| "formats": [ | ||
| "dd/MMM/YYYY:H:m:s Z" | ||
| ] | ||
| } | ||
| }, | ||
| { | ||
| "remove": { | ||
| "field": "nginx.access.time" | ||
| } | ||
| }, | ||
| { | ||
| "user_agent": { | ||
| "field": "nginx.access.agent" | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "nginx.access.agent", | ||
| "target_field": "user_agent.original" | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "user_agent.os", | ||
| "target_field": "user_agent.os.full_name", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "user_agent.os_name", | ||
| "target_field": "user_agent.os.name", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "user_agent.os_major", | ||
| "target_field": "user_agent.os.major", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "user_agent.os_minor", | ||
| "target_field": "user_agent.os.minor", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "user_agent.os_patch", | ||
| "target_field": "user_agent.os.patch", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "geoip": { | ||
| "field": "source.ip", | ||
| "target_field": "source.geo", | ||
| "ignore_missing": true | ||
| } | ||
| } | ||
| ], | ||
| "on_failure": [ | ||
| { | ||
| "set": { | ||
| "field": "error.message", | ||
| "value": "{{ _ingest.on_failure_message }}" | ||
| } | ||
| } | ||
| ] | ||
| } |
Oops, something went wrong.