Map extra threat_response fields from the new default data format. Se…

…t event.kind = alert if an alert ID is present.
elastic · Dec 10, 2024 · d73a9de · d73a9de
1 parent 801b8f8
commit d73a9de
Show file tree

Hide file tree

Showing 10 changed files with 144 additions and 156 deletions.
diff --git a/packages/tanium/data_stream/threat_response/_dev/test/pipeline/test-common-config.yml b/packages/tanium/data_stream/threat_response/_dev/test/pipeline/test-common-config.yml
@@ -4,6 +4,7 @@ fields:
     - preserve_duplicate_custom_fields
 numeric_keyword_fields:
   - tanium.threat_response.id
+  - tanium.threat_response.intel_id
   - tanium.threat_response.match_details.config_id
   - tanium.threat_response.match_details.config_rev_id
   - tanium.threat_response.match_details.finding.whats.intel_intra_ids.id

diff --git a/packages/tanium/data_stream/threat_response/_dev/test/pipeline/test-deep.log-expected.json b/packages/tanium/data_stream/threat_response/_dev/test/pipeline/test-deep.log-expected.json
@@ -9,9 +9,7 @@
                 "category": [
                     "host"
                 ],
-                "kind": [
-                    "event"
-                ],
+                "kind": "alert",
                 "original": "{\"Alert Id\":\"00000000-0000-0000-9a55-325096b39f47\",\"Timestamp\":\"2024-12-02T16:43:02.609Z\",\"Computer Name\":\"hostname.example.com\",\"Computer IP\":\"67.43.156.65\",\"Intel Id\":714,\"Intel Type\":\"openioc\",\"Intel Name\":\"ELK - Linux Test ALert\",\"Intel Labels\":\"\",\"Match Details\":{\"match\":{\"hash\":\"01234567890123456789\",\"type\":\"port\",\"source\":\"threatresponse_database\",\"version\":1,\"properties\":{\"process\":{\"pid\":6460,\"args\":\"\\\"C:\\\\Program Files\\\\Elastic\\\\Agent\\\\data\\\\elastic-agent-8.15.0-asdfas\\\\components\\\\agentbeat.exe\\\" filebeat -E setup.ilm.enabled=false -E setup.template.enabled=false -E management.enabled=true -E management.restart_on_output_change=true -E logging.level=info -E logging.to_stderr=true -E gc_percent=${FILEBEAT_GOGC:100} -E filebeat.config.modules.enabled=false -E logging.event_data.to_stderr=true -E logging.event_data.to_files=false -E http.enabled=true -E http.host=npipe:///asdfasdfasdfasdfasdfasdfasdfasdf.sock -E \\\"path.data=C:\\\\Program Files\\\\Elastic\\\\Agent\\\\data\\\\elastic-agent-8.15.0-asdfas\\\\run\\\\filestream-monitoring\\\"\",\"file\":{\"fullpath\":\"C:\\\\Program Files\\\\Elastic\\\\Agent\\\\data\\\\elastic-agent-8.15.0-25075f\\\\components\\\\agentbeat.exe\"},\"name\":\"agentbeat.exe\",\"ppid\":2536,\"user\":\"NT AUTHORITY\\\\SYSTEM\",\"start_time\":\"2024-12-01T07:33:51.000Z\",\"recorder_table_id\":\"01234567890123456\"},\"local_ip\":\"67.43.156.65\",\"remote_ip\":\"81.2.69.203\",\"local_port\":63123,\"remote_port\":443}},\"finding\":{\"whats\":[{\"source_name\":\"threatresponse_database\",\"intel_intra_ids\":[{\"id_v2\":\"1234567890123456789\"}],\"artifact_activity\":{\"relevant_actions\":[{\"target\":{\"port\":{\"process\":{\"process\":{\"pid\":6460,\"file\":{\"file\":{\"path\":\"C:\\\\Program Files\\\\Elastic\\\\Agent\\\\data\\\\elastic-agent-8.15.0-asdfas\\\\components\\\\agentbeat.exe\",\"signature_data\":{\"issuer\":\"DigiCert Trusted G4 Code Signing RSA4096 SHA384 2021 CA1\",\"status\":1,\"subject\":\"Elasticsearch, Inc.\"}},\"artifact_hash\":\"12345678901234567890\",\"instance_hash\":\"12345678901234562290\"},\"name\":\"agentbeat.exe\",\"user\":{\"user\":{\"name\":\"SYSTEM\",\"domain\":\"NT AUTHORITY\",\"user_id\":\"user-o-1\"}},\"parent\":{\"process\":{\"pid\":2536,\"file\":{\"file\":{\"path\":\"C:\\\\Program Files\\\\Elastic\\\\Agent\\\\data\\\\elastic-agent-8.15.0-asdfas\\\\elastic-agent.exe\",\"signature_data\":{\"issuer\":\"DigiCert Trusted G4 Code Signing RSA4096 SHA384 2021 CA1\",\"status\":1,\"subject\":\"Elasticsearch, Inc.\"}},\"artifact_hash\":\"8123456789012345678\",\"instance_hash\":\"8123456789012345678\"},\"name\":\"elastic-agent.exe\",\"user\":{\"user\":{\"name\":\"SYSTEM\",\"domain\":\"NT AUTHORITY\",\"user_id\":\"uaer-o-1\"}},\"parent\":{\"process\":{\"pid\":824,\"file\":{\"file\":{\"path\":\"C:\\\\Windows\\\\System32\\\\services.exe\",\"signature_data\":{\"status\":11}},\"artifact_hash\":\"3123456789123456789\",\"instance_hash\":\"3123456789123456789\"},\"name\":\"services.exe\",\"user\":{\"user\":{\"name\":\"SYSTEM\",\"domain\":\"NT AUTHORITY\",\"user_id\":\"user-o-1\"}},\"parent\":{\"process\":{\"pid\":680,\"file\":{\"file\":{\"path\":\"C:\\\\Windows\\\\System32\\\\wininit.exe\",\"signature_data\":{\"status\":11}},\"artifact_hash\":\"11234567890123456789\",\"instance_hash\":\"11234567890123456789\"},\"name\":\"wininit.exe\",\"user\":{\"user\":{\"name\":\"SYSTEM\",\"domain\":\"NT AUTHORITY\",\"user_id\":\"user-o-1\"}},\"parent\":{\"process\":{\"handles\":[],\"tanium_recorder_table_id\":\"12345678901234567\"},\"artifact_hash\":\"812345678901234567\",\"instance_hash\":\"5123456789012345678\"},\"handles\":[],\"arguments\":\"wininit.exe\",\"start_time\":\"2024-12-01T07:33:27.000Z\",\"tanium_recorder_table_id\":\"12345678901234567\"},\"artifact_hash\":\"41234567890123456789\",\"instance_hash\":\"11234567890123456789\"},\"handles\":[],\"arguments\":\"C:\\\\Windows\\\\system32\\\\services.exe\",\"start_time\":\"2024-12-01T07:33:27.000Z\",\"tanium_recorder_table_id\":\"71234567890123456\"},\"artifact_hash\":\"8123456789012345678\",\"instance_hash\":\"11234567890123456789\"},\"handles\":[],\"arguments\":\"\\\"C:\\\\Program Files\\\\Elastic\\\\Agent\\\\elastic-agent.exe\\\"\",\"start_time\":\"2024-12-01T07:33:30.000Z\",\"tanium_recorder_table_id\":\"41234567890123456\"},\"artifact_hash\":\"41234556767890123456\",\"instance_hash\":\"31234234563456734564\"},\"handles\":[],\"arguments\":\"\\\"C:\\\\Program Files\\\\Elastic\\\\Agent\\\\data\\\\elastic-agent-8.15.0-asdfas\\\\components\\\\agentbeat.exe\\\" filebeat -E setup.ilm.enabled=false -E setup.template.enabled=false -E management.enabled=true -E management.restart_on_output_change=true -E logging.level=info -E logging.to_stderr=true -E gc_percent=${FILEBEAT_GOGC:100} -E filebeat.config.modules.enabled=false -E logging.event_data.to_stderr=true -E logging.event_data.to_files=false -E http.enabled=true -E http.host=npipe:///asdfasdfasdfasdfasdfasfdasdfasdf.sock -E \\\"path.data=C:\\\\Program Files\\\\Elastic\\\\Agent\\\\data\\\\elastic-agent-8.15.0-asdfas\\\\run\\\\filestream-monitoring\\\"\",\"start_time\":\"2024-12-01T07:33:51.000Z\",\"tanium_recorder_table_id\":\"41234567890123456\"},\"artifact_hash\":\"51234567890123455677\",\"instance_hash\":\"2123456789012345678\"},\"local_ip\":\"67.43.156.65\",\"remote_ip\":\"81.2.69.203\",\"local_port\":63123,\"remote_port\":443,\"connection_time\":\"2024-12-01T16:39:15.000Z\"},\"artifact_hash\":\"61234567890123456789\",\"instance_hash\":\"9123456789012345678\",\"is_intel_target\":true}}]}}],\"domain\":\"threatresponse\",\"hunt_id\":\"hunt:1000123\",\"intel_id\":\"731:3:8bb9caea-3c2c-487c-898c-ebbbf7e4ac55\",\"last_seen\":\"2024-12-01T16:39:19.000Z\",\"threat_id\":\"2123456789012345678\",\"finding_id\":\"5212345678901234567\",\"first_seen\":\"2024-12-01T16:39:19.000Z\",\"source_name\":\"threatresponse_database\",\"system_info\":{\"os\":\"Microsoft Windows Server 2022 Datacenter\",\"bits\":64,\"platform\":\"Windows\",\"patch_level\":\"10.0.20111.0.0\",\"build_number\":\"20321\"},\"reporting_id\":\"hunt:1000111\"},\"intel_id\":714,\"config_id\":1000111,\"config_rev_id\":1},\"MITRE Techniques\":\"[]\",\"Impact Score\":4,\"Link\":\"https://tanium.example.com/#/threatresponse/alerts?guid=b80e010f-bf27-41b8-b028-7f7eb4cbe12b\"}",
                 "type": [
                     "info"
@@ -42,10 +40,16 @@
             ],
             "tanium": {
                 "threat_response": {
+                    "alert_id": "00000000-0000-0000-9a55-325096b39f47",
                     "computer": {
                         "ip": "67.43.156.65",
                         "name": "hostname.example.com"
                     },
+                    "impact_score": 4,
+                    "intel_id": 714,
+                    "intel_name": "ELK - Linux Test ALert",
+                    "intel_type": "openioc",
+                    "link": "https://tanium.example.com/#/threatresponse/alerts?guid=b80e010f-bf27-41b8-b028-7f7eb4cbe12b",
                     "match_details": {
                         "config_id": 1000111,
                         "config_rev_id": 1,

diff --git a/...m/threat_response/_dev/test/pipeline/test-match-details-empty-map-value.log-expected.json b/...m/threat_response/_dev/test/pipeline/test-match-details-empty-map-value.log-expected.json
@@ -9,9 +9,7 @@
                 "category": [
                     "host"
                 ],
-                "kind": [
-                    "event"
-                ],
+                "kind": "alert",
                 "original": "{\"Alert Id\":\"00000000-0000-0000-bff7-f47bab566416\",\"Timestamp\":\"2024-12-01T14:13:14.840Z\",\"Computer Name\":\"hostname.example.com\",\"Computer IP\":\"216.160.83.60\",\"Intel Id\":715,\"Intel Type\":\"openioc\",\"Intel Name\":\"ELK - Linux Test ALert 2 /tmp/iambadvirus.vrs\",\"Intel Labels\":\"\",\"Match Details\":{\"match\":{\"hash\":\"17609926402141399576\",\"type\":\"file\",\"source\":\"at_rest\",\"version\":1,\"properties\":{\"md5\":\"affc5518d1994201d1659890fde69ebb\",\"sha1\":\"3adbbe3ea108260e955fa9ef66355576dabb0e84\",\"sha256\":\"f2f58f9bf1732a38ff996cbfab59ef79b11b67124ed58b8764eb6985d096a23a\",\"fullpath\":\"/tmp/iambadvirus.vrs\"}},\"finding\":{\"whats\":[{\"source_name\":\"at_rest\",\"intel_intra_ids\":[{\"id_v2\":\"51234567891234567890\"}],\"artifact_activity\":{\"relevant_actions\":[{\"target\":{\"file\":{\"hash\":{\"md5\":\"affc5518d1994201d1659890fde69ebb\",\"sha1\":\"3adbbe3ea108260e955fa9ef66355576dabb0e84\",\"sha256\":\"f2f58f9bf1732a38ff996cbfab59ef79b11b67124ed58b8764eb6985d096a23a\"},\"path\":\"/tmp/iambadvirus.vrs\",\"size_bytes\":{},\"magic_number_hex\":{},\"modification_time\":\"2024-12-01T20:25:32.000Z\",\"instance_hash_salt\":\"5432\"},\"artifact_hash\":\"61234567890123456789\",\"instance_hash\":\"7123456778980123456\",\"is_intel_target\":true}}]}}],\"domain\":\"threatresponse\",\"hunt_id\":\"hunt:1000123\",\"intel_id\":\"715:4:8de17746-8210-4ab3-87d2-221ffa7e5dc0\",\"last_seen\":\"2024-12-04T14:10:17.000Z\",\"threat_id\":\"51234567890123456789\",\"finding_id\":\"6123456788901234567\",\"first_seen\":\"2024-12-01T14:10:17.000Z\",\"source_name\":\"at_rest\",\"system_info\":{\"os\":\"Rocky Linux release 9.4 (Blue Onyx)\",\"bits\":64,\"platform\":\"Linux\"},\"reporting_id\":\"hunt:1000123\"},\"intel_id\":715,\"config_id\":1000123,\"config_rev_id\":1},\"MITRE Techniques\":\"[]\",\"Impact Score\":\"\",\"Link\":\"https://tanium.hostname.example.com/#/threatresponse/alerts?guid=00000000-0000-0000-bff7-f47bab566416\"}",
                 "type": [
                     "info"
@@ -47,10 +45,15 @@
             ],
             "tanium": {
                 "threat_response": {
+                    "alert_id": "00000000-0000-0000-bff7-f47bab566416",
                     "computer": {
                         "ip": "216.160.83.60",
                         "name": "hostname.example.com"
                     },
+                    "intel_id": 715,
+                    "intel_name": "ELK - Linux Test ALert 2 /tmp/iambadvirus.vrs",
+                    "intel_type": "openioc",
+                    "link": "https://tanium.hostname.example.com/#/threatresponse/alerts?guid=00000000-0000-0000-bff7-f47bab566416",
                     "match_details": {
                         "config_id": 1000123,
                         "config_rev_id": 1,

diff --git a/.../tanium/data_stream/threat_response/_dev/test/pipeline/test-new-default.log-expected.json b/.../tanium/data_stream/threat_response/_dev/test/pipeline/test-new-default.log-expected.json
@@ -9,9 +9,7 @@
                 "category": [
                     "host"
                 ],
-                "kind": [
-                    "event"
-                ],
+                "kind": "alert",
                 "original": "{\"Alert Id\":\"00000000-0000-0000-a8b0-00cd73a5b9d0\",\"Timestamp\":\"2024-12-01T14:20:00.370Z\",\"Computer Name\":\"hostname.example.com\",\"Computer IP\":\"216.160.83.60\",\"Intel Id\":715,\"Intel Type\":\"openioc\",\"Intel Name\":\"ELK - Linux Test ALert 2 /tmp/iambadvirus.vrs\",\"Intel Labels\":\"\",\"Match Details\":{\"match\":{\"hash\":\"61234567890123456789\",\"type\":\"file\",\"source\":\"at_rest\",\"version\":1,\"properties\":{\"md5\":\"02addf15dea00c97050cc5f08d095e65\",\"sha1\":\"8290fd02d831086c007e95e9ee07481337f4dcef\",\"size\":\"69\",\"sha256\":\"b62aff44a248c9934fc9c0daefe3926c347a260bdb4ef59d0a77e6db4be9c786\",\"fullpath\":\"/tmp/verybadvirus.vrs\"}},\"finding\":{\"whats\":[{\"source_name\":\"at_rest\",\"intel_intra_ids\":[{\"id_v2\":\"11234567890123456789\"}],\"artifact_activity\":{\"relevant_actions\":[{\"target\":{\"file\":{\"hash\":{\"md5\":\"02addf15dea00c97050cc5f08d095e65\",\"sha1\":\"8290fd02d831086c007e95e9ee07481337f4dcef\",\"sha256\":\"b62aff44a248c9934fc9c0daefe3926c347a260bdb4ef59d0a77e6db4be9c786\"},\"path\":\"/tmp/verybadvirus.vrs\",\"size_bytes\":\"69\",\"magic_number_hex\":\"23098420\",\"modification_time\":\"2024-12-01T17:42:10.000Z\",\"instance_hash_salt\":\"1234\"},\"artifact_hash\":\"61234567890123456789\",\"instance_hash\":\"1123456789012345678\",\"is_intel_target\":true}}]}}],\"domain\":\"threatresponse\",\"hunt_id\":\"hunt:1000123\",\"intel_id\":\"715:4:b70b41e7-e698-451a-a7e8-b7524a6e6c3c\",\"last_seen\":\"2024-12-01T14:15:25.000Z\",\"threat_id\":\"56112345667882345566\",\"finding_id\":\"712345678901234567\",\"first_seen\":\"2024-12-01T14:15:25.000Z\",\"source_name\":\"at_rest\",\"system_info\":{\"os\":\"Rocky Linux release 9.4 (Blue Onyx)\",\"bits\":64,\"platform\":\"Linux\"},\"reporting_id\":\"hunt:1000123\"},\"intel_id\":715,\"config_id\":1000123,\"config_rev_id\":1},\"MITRE Techniques\":\"[]\",\"Impact Score\":\"\",\"Link\":\"https://tanium.example.com/#/threatresponse/alerts?guid=00000000-0000-0000-a8b0-00cd73a5b9d0\"}",
                 "type": [
                     "info"
@@ -47,10 +45,15 @@
             ],
             "tanium": {
                 "threat_response": {
+                    "alert_id": "00000000-0000-0000-a8b0-00cd73a5b9d0",
                     "computer": {
                         "ip": "216.160.83.60",
                         "name": "hostname.example.com"
                     },
+                    "intel_id": 715,
+                    "intel_name": "ELK - Linux Test ALert 2 /tmp/iambadvirus.vrs",
+                    "intel_type": "openioc",
+                    "link": "https://tanium.example.com/#/threatresponse/alerts?guid=00000000-0000-0000-a8b0-00cd73a5b9d0",
                     "match_details": {
                         "config_id": 1000123,
                         "config_rev_id": 1,