Nats Filebeat (#10514)

filebeat/docs/fields.asciidoc

@@ -31,6 +31,7 @@ grouped in the following categories:
 * <<exported-fields-logstash>>
 * <<exported-fields-mongodb>>
 * <<exported-fields-mysql>>
+* <<exported-fields-nats>>
 * <<exported-fields-netflow>>
 * <<exported-fields-nginx>>
 * <<exported-fields-osquery>>
@@ -7385,6 +7386,26 @@ alias to: source.ip
 
 --
 
+[[exported-fields-nats]]
+== nats fields
+
+Module for parsing NATS log files.
+
+
+
+[float]
+== nats fields
+
+Fields from NATS logs.
+
+
+
+[float]
+== log fields
+
+Nats log files
+
+
 [[exported-fields-netflow]]
 == NetFlow fields
 

filebeat/docs/modules/nats.asciidoc

@@ -0,0 +1,52 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-nats]]
+:modulename: nats
+:has-dashboards: false
+
+== nats module
+
+This is the nats module.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+The +{modulename}+ module was tested with logs from version v1.4.0.
+
+
+include::../include/running-modules.asciidoc[]
+
+[float]
+=== Example dashboard
+
+
+include::../include/configuring-intro.asciidoc[]
+
+
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+
+[float]
+==== `log` log fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-nats,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -15,6 +15,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-logstash>>
   * <<filebeat-module-mongodb>>
   * <<filebeat-module-mysql>>
+  * <<filebeat-module-nats>>
   * <<filebeat-module-nginx>>
   * <<filebeat-module-osquery>>
   * <<filebeat-module-postgresql>>
@@ -41,6 +42,7 @@ include::modules/kibana.asciidoc[]
 include::modules/logstash.asciidoc[]
 include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]
+include::modules/nats.asciidoc[]
 include::modules/nginx.asciidoc[]
 include::modules/osquery.asciidoc[]
 include::modules/postgresql.asciidoc[]

filebeat/filebeat.reference.yml

@@ -290,6 +290,16 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+#--------------------------------- Nats Module ---------------------------------
+- module: nats
+  # All logs
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #-------------------------------- Nginx Module --------------------------------
 #- module: nginx
   # Access logs

filebeat/module/nats/_meta/config.yml

@@ -0,0 +1,8 @@
+- module: nats
+  # All logs
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/nats/_meta/docs.asciidoc

@@ -0,0 +1,39 @@
+:modulename: nats
+:has-dashboards: false
+
+== nats module
+
+This is the nats module.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+The +{modulename}+ module was tested with logs from version v1.4.0.
+
+
+include::../include/running-modules.asciidoc[]
+
+[float]
+=== Example dashboard
+
+
+include::../include/configuring-intro.asciidoc[]
+
+
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+
+[float]
+==== `log` log fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:

filebeat/module/nats/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: nats
+  title: "nats"
+  description: >
+    Module for parsing NATS log files.
+  release: beta
+  fields:
+    - name: nats
+      type: group
+      description: >
+        Fields from NATS logs.
+      fields:

filebeat/module/nats/log/_meta/fields.yml

@@ -0,0 +1,6 @@
+- name: log
+  type: group
+  description: >
+    Nats log files
+  release: beta
+  fields:

filebeat/module/nats/log/config/log.yml

@@ -0,0 +1,6 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]

filebeat/module/nats/log/ingest/pipeline.json

@@ -0,0 +1,60 @@
+{
+  "description": "Pipeline for parsing nats log logs",
+    "processors": [{
+            "grok": {
+                "field": "message",
+                "patterns":[
+                    "\\[%{POSINT:process.pid}\\]( %{NATSTIME:nats.log.timestamp})? \\[%{NATSLOGLEVEL:log.level}\\] %{GREEDYDATA:message}"
+                ],
+                "pattern_definitions": {
+                    "NATSTIME": "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}",
+                    "NATSLOGLEVEL":"(INF|DBG|WRN|ERR|FTL|TRC)"
+                },
+                "ignore_missing": true
+            }
+        },
+        {
+            "script": {
+                "lang": "painless",
+                "source": "if (ctx.log.level == params.inf) {\n          ctx.log.level = params.info;\n        } else if (ctx.log.level == params.dbg) {\n          ctx.log.level = params.debug;\n        } else if (ctx.log.level == params.wrn) {\n          ctx.log.level = params.warning;\n        } else if (ctx.log.level == params.err) {\n          ctx.log.level = params.error;\n        } else if (ctx.log.level == params.ftl) {\n          ctx.log.level = params.fatal;\n        } else if (ctx.log.level == params.trc) {\n          ctx.log.level = params.trace;\n        }",
+                "params": {
+                    "inf": "INF",
+                    "info": "info",
+                    "dbg": "DBG",
+                    "debug": "debug",
+                    "wrn": "WRN",
+                    "warning": "warning",
+                    "err": "ERR",
+                    "error": "error",
+                    "ftl": "FTL",
+                    "fatal": "fatal",
+                    "trc": "TRC",
+                    "trace": "trace"
+                }
+            }
+        },
+        {
+            "rename": {
+                "field": "@timestamp",
+                "target_field": "event.created"
+            }
+        },
+        {
+            "date": {
+                "field": "nats.log.timestamp",
+                "target_field": "@timestamp",
+                "formats": ["yyyy/MM/dd HH:mm:ss.SSSSSS"]
+            }
+        },
+        {
+            "remove": {
+                "field": "nats.log.timestamp"
+            }
+        }],
+    "on_failure" : [{
+        "set" : {
+            "field" : "error.message",
+            "value" : "{{ _ingest.on_failure_message }}"
+        }
+    }]
+}

filebeat/module/nats/log/manifest.yml

@@ -0,0 +1,15 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      # This is an example location of where a user can
+      # set the nats server to log. By default nats
+      # logs only in stdout. In order to set a nats server
+      # to log in specific file use -l flag, for instance:
+      # ./gnatsd -DV -l /var/log/nats/nats.log
+      # See more on https://nats.io/documentation/server/gnatsd-logging/
+      - /var/log/nats/nats.log*
+
+ingest_pipeline: ingest/pipeline.json
+input: config/log.yml

filebeat/module/nats/log/test/test.log

@@ -0,0 +1,9 @@
+[1] 2019/02/06 07:19:40.624334 [INF] Starting nats-server version 1.3.0
+[1] 2019/02/06 07:19:40.624547 [INF] Git commit [eed4fbc]
+[1] 2019/02/06 07:19:40.624674 [INF] Listening for client connections on 0.0.0.0:4222
+[1] 2019/02/06 07:19:40.624690 [INF] Server is ready
+[1] 2019/02/06 07:20:08.508891 [DBG] 172.18.0.1:38630 - cid:1 - Client connection created
+[1] 2019/02/06 07:20:08.510296 [TRC] 172.18.0.1:38630 - cid:1 - ->> [CONNECT {"verbose":false,"pedantic":false,"tls_required":false,"name":"NATS Benchmark","lang":"go","version":"1.7.0","protocol":1,"echo":true}]
+[1] 2019/02/06 07:20:08.512052 [TRC] 172.18.0.1:38630 - cid:1 - ->> [SUB foo  1]
+[1] 2019/02/06 07:20:08.512128 [TRC] 172.18.0.1:38630 - cid:1 - ->> [PING]
+[1] 2019/02/06 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - <<- [PONG]