Populate more ECS fields in the Suricata module (#10006)

* Populate more ECS fields in the Suricata module

A few more ECS fields are populated by the ingest pipeline that enriches
Suricata's eve.json events.

Additions:

- http.request.referrer (from suricata.eve.http.http_refer)

- event.action (from suricata.eve.alert.category)
  describes the action that caused the event.
  Examples: "Attempted Denial of Service", "Successful Administrator Privilege Gain"

- event.outcome (from suricata.eve.alert.action)
  Possible values: "allowed", "blocked"

- event.severity (from suricata.eve.alert.severity)
  Possible values: 1, 2 or 3.

- network.transport (from suricata.eve.proto)
  Examples: "tcp", "udp", "ipv6-icmp"

* Use message for suricata.eve.alert.category

Instead of event.action, which is expected to have a fixed set of
enumeration values.

* Populate destination.domain

When http.hostname is present.

* Populate event.{start,end,duration}

* populate network.protocol

* url.hostname is url.domain

* Populate url.path, url.fragment, url.query

From http.url

* Lowercase http request method

* Source/Destination and aggregated counters

This assumes client=source server=destination.

Populates
- source.{packets|bytes}
- destination.{packets|bytes}
- network.{packets|bytes}

* Updated golden files

* Populate ECS field `http.response.body.bytes`

* Use grok pattern to parse url fields

Replace ugly painless code.

* Avoid pairs of convert/lowercase

Lowercase processor can have a target field so its not neccesary to copy
the field in a previous step.

* Cleanup painless script

* Fix golden data

* Fix golden data (2)

* Copy timestamp to event.end instead of parsing date again

x-pack/filebeat/module/suricata/eve/ingest/pipeline.json

@@ -33,27 +33,58 @@
       ,"ignore_missing": true
       }
     }
+  , {"lowercase":
+       {"field": "suricata.eve.http.http_method"
+       ,"target_field": "http.request.method"
+       ,"ignore_missing": true
+      }
+    }
   , {"convert":
-      {"field": "suricata.eve.http.http_method"
-      ,"target_field": "http.request.method"
+      {"field": "suricata.eve.http.status"
+      ,"target_field": "http.response.status_code"
       ,"type": "string"
       ,"ignore_missing": true
       }
     }
   , {"convert":
-      {"field": "suricata.eve.http.status"
-      ,"target_field": "http.response.status_code"
+      {"field": "suricata.eve.http.hostname"
+      ,"target_field": "url.domain"
       ,"type": "string"
       ,"ignore_missing": true
       }
     }
+  , { "grok":
+      { "field": "suricata.eve.http.url"
+      , "patterns": ["%{PATH:url.path}(?:\\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?"]
+      , "ignore_missing": true
+      , "pattern_definitions":
+        { "PATH": "[^?#]*"
+        , "QUERY": "[^#]*"
+        , "ANY": ".*"
+        }
+      }
+    }
   , {"convert":
       {"field": "suricata.eve.http.hostname"
-      ,"target_field": "url.hostname"
+      ,"target_field": "destination.domain"
       ,"type": "string"
       ,"ignore_missing": true
       }
     }
+  , {"convert":
+      {"field": "suricata.eve.http.http_refer"
+      ,"target_field": "http.request.referrer"
+      ,"type": "string"
+      ,"ignore_missing": true
+      }
+    }
+  , {"convert":
+      {"field": "suricata.eve.http.length"
+      ,"target_field": "http.response.body.bytes"
+      ,"type": "integer"
+      ,"ignore_missing": true
+      }
+    }
   , {"convert":
       {"field": "suricata.eve.fileinfo.filename"
       ,"target_field": "file.path"
@@ -85,15 +116,93 @@
 
   , { "lowercase":
       { "field": "suricata.eve.event_type"
+      , "target_field": "event.type"
       , "ignore_missing": true
       }
     }
-  , { "set":
-      { "field": "event.type"
-      , "value": "{{suricata.eve.event_type}}"
+  , {"convert":
+      {"field": "suricata.eve.alert.category"
+      ,"target_field": "message"
+      ,"type": "string"
+      ,"ignore_missing": true
+      }
+    }
+  , {"convert":
+      {"field": "suricata.eve.alert.action"
+      ,"target_field": "event.outcome"
+      ,"type": "string"
+      ,"ignore_missing": true
+      }
+    }
+  , {"convert":
+      {"field": "suricata.eve.alert.severity"
+      ,"target_field": "event.severity"
+      ,"type": "integer"
+      ,"ignore_missing": true
+      }
+    }
+  , {"convert":
+      {"field": "suricata.eve.flow.pkts_toclient"
+      ,"target_field": "destination.packets"
+      ,"type": "integer"
+      ,"ignore_missing": true
+      }
+    }
+  , {"convert":
+      {"field": "suricata.eve.flow.pkts_toserver"
+      ,"target_field": "source.packets"
+      ,"type": "integer"
+      ,"ignore_missing": true
+      }
+    }
+  , {"convert":
+      {"field": "suricata.eve.flow.bytes_toclient"
+      ,"target_field": "destination.bytes"
+      ,"type": "integer"
+      ,"ignore_missing": true
+      }
+    }
+  , {"convert":
+      {"field": "suricata.eve.flow.bytes_toserver"
+      ,"target_field": "source.bytes"
+      ,"type": "integer"
+      ,"ignore_missing": true
+      }
+    }
+  , { "script":
+      { "lang": "painless"
+      , "source": "long getOrZero(def map, def key) { if(map!=null && map[key]!=null) { return map[key]; } return 0; } def network=ctx['network'], source=ctx['source'], dest=ctx['destination']; def sp=getOrZero(source,'packets'), sb=getOrZero(source,'bytes'), dp=getOrZero(dest,'packets'), db=getOrZero(dest,'bytes'); if(sb+db+sp+dp > 0){if (network==null){network=new HashMap(); ctx['network']=network; } if(sb+db>0) network['bytes'] = sb+db; if(sp+dp>0) network['packets'] = sp+dp; }"
+      }
+    }
+  , {"date":
+      {"field": "suricata.eve.flow.start"
+      ,"target_field": "event.start"
+      ,"formats": ["ISO8601"]
+      ,"ignore_failure": true
+      }
+    }
+  , {"set":
+      {"field": "event.end"
+      ,"value": "{{@timestamp}}"
+      }
+    }
+  , { "script":
+      { "lang": "painless"
+      , "source": "Instant ins(def d){try{return Instant.parse(d);}catch(Exception e){return null;}}def ev=ctx['event'];if(ev!=null){def start=ins(ev['start']); def end=ins(ev['end']); if(start!=null && end!=null && !start.isAfter(end)) {ev['duration'] = Duration.between(start,end).toNanos();}}"
+      }
+    }
+  , { "lowercase":
+      { "field": "suricata.eve.proto"
+      , "target_field": "network.transport"
+      , "ignore_missing": true
+      }
+    }
+  , { "lowercase":
+      { "field": "suricata.eve.app_proto"
+      , "target_field": "network.protocol"
+      , "ignore_missing": true
       }
     }
-
   , { "user_agent":
       { "field": "user_agent.original"
       , "target_field": "user_agent"