Improve ECS categorization field mappings for nginx module (elastic#1…

…7844)

- access
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + lowercase http.request.method
  + improve grok to not populate empty fields
  + related.ip
  + related.users
- error
  + event.kind
  + event.category
  + event.outcome
- ingress_controller
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + lowercase http.request.method
  + improve grok to not populate empty fields
  + related.ip
  + related.users

Closes elastic#16174

CHANGELOG.next.asciidoc

@@ -269,6 +269,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729]
 - Improve ECS categorization field mappings in misp module. {issue}16026[16026] {pull}17344[17344]
 - Added Unix stream socket support as an input source and a syslog input source. {pull}17492[17492]
+- Improve ECS categorization field mappings for nginx module. {issue}16174[16174] {pull}17844[17844]
 
 *Heartbeat*
 

filebeat/module/nginx/access/ingest/pipeline.yml

@@ -0,0 +1,167 @@
+description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent
+  plugins.
+processors:
+- grok:
+    field: message
+    patterns:
+    - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address})
+      - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}"
+      %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}
+      "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})"
+    pattern_definitions:
+      NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?
+      NGINX_NOTSEPARATOR: "[^\t ,:]+"
+      NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))*
+    ignore_missing: true
+- grok:
+    field: nginx.access.info
+    patterns:
+    - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}'
+    - ""
+    ignore_missing: true
+- remove:
+    field: nginx.access.info
+- split:
+    field: nginx.access.remote_ip_list
+    separator: '"?,?\s+'
+    ignore_missing: true
+- split:
+    field: nginx.access.origin
+    separator: '"?,?\s+'
+    ignore_missing: true
+- set:
+    field: source.address
+    if: ctx.source?.address == null
+    value: ""
+- script:
+    if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0
+    lang: painless
+    source: >-
+      boolean isPrivate(def dot, def ip) {
+        try {
+          StringTokenizer tok = new StringTokenizer(ip, dot);
+          int firstByte = Integer.parseInt(tok.nextToken());
+          int secondByte = Integer.parseInt(tok.nextToken());
+          if (firstByte == 10) {
+            return true;
+          }
+          if (firstByte == 192 && secondByte == 168) {
+            return true;
+          }
+          if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {
+            return true;
+          }
+          if (firstByte == 127) {
+            return true;
+          }
+          return false;
+        }
+        catch (Exception e) {
+          return false;
+        }
+      }
+      try {
+        ctx.source.address = null;
+        if (ctx.nginx.access.remote_ip_list == null) {
+          return;
+        }
+        def found = false;
+        for (def item : ctx.nginx.access.remote_ip_list) {
+          if (!isPrivate(params.dot, item)) {
+            ctx.source.address = item;
+            found = true;
+            break;
+          }
+        }
+        if (!found) {
+          ctx.source.address = ctx.nginx.access.remote_ip_list[0];
+        }
+      }
+      catch (Exception e) {
+        ctx.source.address = null;
+      }
+    params:
+      dot: .
+- remove:
+    field: source.address
+    if: ctx.source.address == null
+- grok:
+    field: source.address
+    patterns:
+    - ^%{IP:source.ip}$
+    ignore_failure: true
+- remove:
+    field: message
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- date:
+    field: nginx.access.time
+    target_field: '@timestamp'
+    formats:
+    - dd/MMM/yyyy:H:m:s Z
+    on_failure:
+    - append:
+        field: error.message
+        value: '{{ _ingest.on_failure_message }}'
+- remove:
+    field: nginx.access.time
+- user_agent:
+    field: user_agent.original
+    ignore_missing: true
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_missing: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: event.category
+    value: web
+- append:
+    field: event.type
+    value: access
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
+- set:
+    field: event.outcome
+    value: failure
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"      
+- lowercase:
+    field: http.request.method
+    ignore_missing: true
+- append:
+    field: related.ip
+    value: "{{source.ip}}"
+    if: "ctx?.source?.ip != null"
+- append:
+    field: related.ip
+    value: "{{destination.ip}}"
+    if: "ctx?.destination?.ip != null"
+- append:
+    field: related.user
+    value: "{{user.name}}"
+    if: "ctx?.user?.name != null"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/nginx/access/manifest.yml

@@ -9,7 +9,7 @@ var:
     os.windows:
       - c:/programdata/nginx/logs/*access.log*
 
-ingest_pipeline: ingest/default.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/nginx-access.yml
 
 requires.processors: