Add Zeek NTP and Signature data streams

elastic · Aug 26, 2021 · 3eac588 · 3eac588
1 parent 51c6299
commit 3eac588
Show file tree

Hide file tree

Showing 41 changed files with 2,436 additions and 74 deletions.
diff --git a/packages/zeek/_dev/build/docs/README.md b/packages/zeek/_dev/build/docs/README.md
@@ -124,6 +124,13 @@ Zeek notices.
 
 {{fields "notice"}}
 
+### ntp
+
+The `ntp` dataset collects the Zeek ntp.log file, which contains
+NTP data.
+
+{{fields "ntp"}}
+
 ### ntlm
 
 The `ntlm` dataset collects the Zeek ntlm.log file, which contains NT
@@ -166,6 +173,13 @@ Remote Framebuffer (RFB) data.
 
 {{fields "rfb"}}
 
+### signature
+
+The `signature` dataset collects the Zeek signature.log file, which contains
+Zeek signature matches.
+
+{{fields "signature"}}
+
 ### sip
 
 The `sip` dataset collects the Zeek sip.log file, which contains SIP

diff --git a/packages/zeek/_dev/deploy/docker/http-mock-config.yml b/packages/zeek/_dev/deploy/docker/http-mock-config.yml
@@ -339,6 +339,26 @@ rules:
             - "application/json"
         body: |-
           {"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1508959117.814467,\"uid\":\"CHphiNUKDC20fsy09\",\"id.orig_h\":\"192.168.10.50\",\"id.orig_p\":46785,\"id.resp_h\":\"192.168.10.31\",\"id.resp_p\":445,\"username\":\"JeffV\",\"hostname\":\"ybaARon55QykXrgu\",\"domainname\":\"contoso.local\",\"server_nb_computer_name\":\"VICTIM-PC\",\"server_dns_computer_name\":\"Victim-PC.contoso.local\",\"server_tree_name\":\"contoso.local\"}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/zeek/ntlm.log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+  - path: /services/search/jobs/export
+    user: test
+    password: test
+    methods:
+      - POST
+    query_params:
+      index_earliest: "{index_earliest:[0-9]+}"
+      index_latest: "{index_latest:[0-9]+}"
+      output_mode: json
+      search: 'search sourcetype="ntp-*" | streamstats max(_indextime) AS max_indextime'
+    request_headers:
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1602116947.977,\"uid\":\"CqlPpF1AQVLMPgGiL5\",\"id.orig_h\":\"130.118.205.62\",\"id.orig_p\":38461,\"id.resp_h\":\"208.79.89.249\",\"id.resp_p\":123,\"version\":4,\"mode\":3,\"stratum\":0,\"poll\":1,\"precision\":1,\"root_delay\":0,\"root_disp\":0,\"ref_id\":\"\\\\x00\\\\x00\\\\x00\\\\x00\",\"ref_time\":0,\"org_time\":0,\"rec_time\":0,\"xmt_time\":1602116947.215,\"num_exts\":0}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/zeek/ntp.log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
   - path: /services/search/jobs/export
     user: test
     password: test
@@ -439,6 +459,26 @@ rules:
             - "application/json"
         body: |-
           {"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1328632534.517208,\"uid\":\"CXoIzM3wH3fUwXtKN1\",\"id.orig_h\":\"192.168.1.123\",\"id.orig_p\":58102,\"id.resp_h\":\"192.168.1.10\",\"id.resp_p\":5900,\"client_major_version\":\"003\",\"client_minor_version\":\"008\",\"server_major_version\":\"003\",\"server_minor_version\":\"008\",\"authentication_method\":\"VNC\",\"auth\":true,\"share_flag\":false,\"desktop_name\":\"\\u00a0\",\"width\":800,\"height\":600}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/zeek/rfb.log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+  - path: /services/search/jobs/export
+    user: test
+    password: test
+    methods:
+      - POST
+    query_params:
+      index_earliest: "{index_earliest:[0-9]+}"
+      index_latest: "{index_latest:[0-9]+}"
+      output_mode: json
+      search: 'search sourcetype="signature-*" | streamstats max(_indextime) AS max_indextime'
+    request_headers:
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1611852809.869245,\"uid\":\"CbjAXE4CBxJ8W7VoJg\",\"src_addr\":\"124.51.137.154\",\"src_port\":51617,\"dst_addr\":\"160.218.27.63\",\"dst_port\":445,\"note\":\"Signatures::Sensitive_Signature\",\"sig_id\":\"my-second-sig\",\"event_msg\":\"124.51.137.154:TCP traffic\",\"sub_msg\":\"\"}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/zeek/signature.log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
   - path: /services/search/jobs/export
     user: test
     password: test

diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/ntp.log b/packages/zeek/_dev/deploy/docker/sample_logs/ntp.log
@@ -0,0 +1,2 @@
+{"ts":1602116947.977,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":3,"stratum":0,"poll":1,"precision":1,"root_delay":0,"root_disp":0,"ref_id":"\\x00\\x00\\x00\\x00","ref_time":0,"org_time":0,"rec_time":0,"xmt_time":1602116947.215,"num_exts":0}
+{"ts":1602116948.081,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":4,"stratum":2,"poll":8,"precision":5.960464477539063e-8,"root_delay":0.00921630859375,"root_disp":0.0212249755859375,"ref_id":"127.67.113.92","ref_time":1602116655.942,"org_time":1602116947.215,"rec_time":1602116947.964,"xmt_time":1602116947.964,"num_exts":0}
diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/signature.log b/packages/zeek/_dev/deploy/docker/sample_logs/signature.log
@@ -0,0 +1 @@
+{"ts": 1611852809.869245,"uid": "CbjAXE4CBxJ8W7VoJg","src_addr": "124.51.137.154","src_port": 51617,"dst_addr": "160.218.27.63","dst_port": 445,"note": "Signatures::Sensitive_Signature","sig_id": "my-second-sig","event_msg": "124.51.137.154: TCP traffic","sub_msg": ""}
diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Add Sigature and NTP data streams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1515
 - version: "1.2.2"
   changes:
     - description: Convert to generated ECS fields
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"ts":1602116947.977,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":3,"stratum":0,"poll":1,"precision":1,"root_delay":0,"root_disp":0,"ref_id":"\\x00\\x00\\x00\\x00","ref_time":0,"org_time":0,"rec_time":0,"xmt_time":1602116947.215,"num_exts":0}
		{"ts":1602116948.081,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":4,"stratum":2,"poll":8,"precision":5.960464477539063e-8,"root_delay":0.00921630859375,"root_disp":0.0212249755859375,"ref_id":"127.67.113.92","ref_time":1602116655.942,"org_time":1602116947.215,"rec_time":1602116947.964,"xmt_time":1602116947.964,"num_exts":0}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"ts": 1611852809.869245,"uid": "CbjAXE4CBxJ8W7VoJg","src_addr": "124.51.137.154","src_port": 51617,"dst_addr": "160.218.27.63","dst_port": 445,"note": "Signatures::Sensitive_Signature","sig_id": "my-second-sig","event_msg": "124.51.137.154: TCP traffic","sub_msg": ""}