From 9f780357e9dfd7dd24a1fcd537d85f7b1f88dd64 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 10 Dec 2020 15:55:49 +0100 Subject: [PATCH] MISP improvements: - Add id based on body uuid - Add pagination and cursor - Migrate to httpjson v2 --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/modules/misp.asciidoc | 4 +- x-pack/filebeat/filebeat.reference.yml | 9 ++--- x-pack/filebeat/module/misp/_meta/config.yml | 9 ++--- .../filebeat/module/misp/_meta/docs.asciidoc | 4 +- .../module/misp/threat/config/input.yml | 37 ++++++++++++++----- .../module/misp/threat/config/pipeline.js | 18 +++++++-- .../filebeat/module/misp/threat/manifest.yml | 30 +++------------ .../misp/threat/test/misp-test.json.log | 2 +- .../test/misp-test.json.log-expected.json | 11 ++++-- x-pack/filebeat/modules.d/misp.yml.disabled | 9 ++--- 11 files changed, 70 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0d1563a2c4b0..e5e93a3a9eb1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -768,6 +768,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add the ability to override `network.direction` based on interfaces in Fortinet/firewall fileset. {pull}23072[23072] - Add `network.direction` override by specifying `internal_networks` in gcp module. {pull}23081[23081] - Migrate okta to httpjson v2 config {pull}23059[23059] +- Misp improvements: Migration to httpjson v2 config, pagination and deduplication ID {pull}23070[23070] *Heartbeat* diff --git a/filebeat/docs/modules/misp.asciidoc b/filebeat/docs/modules/misp.asciidoc index 53d44ba51376..4eebc6150c41 100644 --- a/filebeat/docs/modules/misp.asciidoc +++ b/filebeat/docs/modules/misp.asciidoc @@ -17,9 +17,9 @@ This is a filebeat module for reading threat intel information from the MISP pla The configuration in the config.yml file uses the following format: * var.api_key: specifies the API key to access MISP. - * var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute". + * var.http_request_body: an object containing any parameter that needs to be sent to the search API. Default: `limit: 1000` * var.url: URL of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch" - + include::../include/gs-link.asciidoc[] [float] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index fdd805603a4e..840f79bf2388 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1340,16 +1340,13 @@ filebeat.modules: #var.api_key # Array object in MISP response - #var.json_objects_array + #var.http_request_body.limit: 1000 # URL of the MISP REST API #var.url - + # You can also pass SSL options. For example: - #var.ssl: |- - # { - # verification_mode: none - # } + #var.ssl.verification_mode: none #------------------------------- Mongodb Module ------------------------------- #- module: mongodb diff --git a/x-pack/filebeat/module/misp/_meta/config.yml b/x-pack/filebeat/module/misp/_meta/config.yml index 5353bf28b48c..67c7dc566d89 100644 --- a/x-pack/filebeat/module/misp/_meta/config.yml +++ b/x-pack/filebeat/module/misp/_meta/config.yml @@ -5,13 +5,10 @@ #var.api_key # Array object in MISP response - #var.json_objects_array + #var.http_request_body.limit: 1000 # URL of the MISP REST API #var.url - + # You can also pass SSL options. For example: - #var.ssl: |- - # { - # verification_mode: none - # } + #var.ssl.verification_mode: none diff --git a/x-pack/filebeat/module/misp/_meta/docs.asciidoc b/x-pack/filebeat/module/misp/_meta/docs.asciidoc index c8082cb9ee51..ea0e671c06b4 100644 --- a/x-pack/filebeat/module/misp/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/misp/_meta/docs.asciidoc @@ -12,9 +12,9 @@ This is a filebeat module for reading threat intel information from the MISP pla The configuration in the config.yml file uses the following format: * var.api_key: specifies the API key to access MISP. - * var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute". + * var.http_request_body: an object containing any parameter that needs to be sent to the search API. Default: `limit: 1000` * var.url: URL of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch" - + include::../include/gs-link.asciidoc[] [float] diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml index 25fb6f950b65..81deb7e5aa5e 100644 --- a/x-pack/filebeat/module/misp/threat/config/input.yml +++ b/x-pack/filebeat/module/misp/threat/config/input.yml @@ -1,16 +1,35 @@ {{ if eq .input "httpjson" }} type: httpjson -api_key: {{ .api_key }} -http_client_timeout: {{ .http_client_timeout }} -http_method: {{ .http_method }} -http_headers: {{ .http_headers }} -http_request_body: {{ .http_request_body }} +config_version: "2" interval: {{ .interval }} -json_objects_array: {{ .json_objects_array }} -pagination: {{ .pagination }} -url: {{ .url }} -ssl: {{ .ssl }} + +request.method: POST +request.ssl: {{ .ssl | tojson }} +request.url: {{ .url }} +request.timeout: {{ .http_client_timeout }} +request.body: {{ .http_request_body | tojson }} +request.body.returnFormat: json +request.transforms: + - set: + target: header.Authorization + value: {{.api_key}} + - set: + target: body.timestamp + value: "[[.cursor.timestamp]]" + +response.split: + target: body.response.Attribute + +response.request_body_on_pagination: true +response.pagination: + - set: + target: body.page + value: "[[add .last_response.page 1]]" + +cursor: + timestamp: + value: "[[.last_event.timestamp]]" {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/misp/threat/config/pipeline.js b/x-pack/filebeat/module/misp/threat/config/pipeline.js index d5b2898b2fe0..472eb3597d68 100644 --- a/x-pack/filebeat/module/misp/threat/config/pipeline.js +++ b/x-pack/filebeat/module/misp/threat/config/pipeline.js @@ -5,11 +5,19 @@ var threat = (function () { var processor = require("processor"); + var copyToOriginal = function (evt) { + evt.Put("event.original", evt.Get("message")); + }; + var decodeJson = new processor.DecodeJSONFields({ fields: ["message"], target: "json", }); + var setID = function (evt) { + evt.Put("@metadata._id", evt.Get("event.id")); + }; + var categorizeEvent = new processor.AddFields({ target: "event", fields: { @@ -73,11 +81,11 @@ var threat = (function () { break; case 'btc': attackPattern = '[' + 'bitcoin:address = ' + '\'' + v + '\'' + ']'; - attackPatternKQL = 'bitcoin.address: ' + '"' + v + '"'; + attackPatternKQL = 'bitcoin.address: ' + '"' + v + '"'; break; case "domain": attackPattern = '[' + 'dns:question:name = ' + '\'' + v + '\'' + ' OR url:domain = ' + '\'' + v + '\'' + ' OR source:domain = ' + '\'' + v + '\'' + ' OR destination:domain = ' + '\'' + v + '\'' + ']'; - attackPatternKQL = 'dns.question.name: ' + '"' + v + '"' + ' OR url.domain: ' + '"' + v + '"' + ' OR source.domain: ' + '"' + v + '"' + ' OR destination.domain: ' + '"' + v + '"'; + attackPatternKQL = 'dns.question.name: ' + '"' + v + '"' + ' OR url.domain: ' + '"' + v + '"' + ' OR source.domain: ' + '"' + v + '"' + ' OR destination.domain: ' + '"' + v + '"'; break; case "domain|ip": arr = v.split("|"); @@ -186,7 +194,7 @@ var threat = (function () { attackPattern = '[' + 'file:hash:sha256 = ' + '\'' + v + '\'' + ']'; attackPatternKQL = 'file.hash.sha256: ' + '"' + v + '"'; evt.Put("file.hash.sha256", v); - break; + break; case "sha512": attackPattern = '[' + 'file:hash:sha512 = ' + '\'' + v + '\'' + ']'; attackPatternKQL = 'file.hash.sha512: ' + '"' + v + '"'; @@ -200,7 +208,7 @@ var threat = (function () { case 'yara': attackPattern = '[' + 'yara:rule = ' + '\'' + v + '\'' + ']'; attackPatternKQL = 'yara.rule: ' + '"' + v + '"'; - break; + break; } if (attackPattern == undefined || attackPatternKQL == undefined) { evt.Put("error.message", 'Unsupported type: ' + indicator_type); @@ -210,10 +218,12 @@ var threat = (function () { }; var pipeline = new processor.Chain() + .Add(copyToOriginal) .Add(decodeJson) .Add(categorizeEvent) .Add(setThreatFeedField) .Add(convertFields) + .Add(setID) .Add(setAttackPattern) .Add(copyTags) .Build(); diff --git a/x-pack/filebeat/module/misp/threat/manifest.yml b/x-pack/filebeat/module/misp/threat/manifest.yml index 41a85a6f74db..63e3ef6db6e0 100644 --- a/x-pack/filebeat/module/misp/threat/manifest.yml +++ b/x-pack/filebeat/module/misp/threat/manifest.yml @@ -4,35 +4,15 @@ var: - name: input default: httpjson - name: api_key - default: "" - - name: http_client_timeout - default: 60 - - name: http_method - default: GET - - name: http_headers - default: |- - {} - name: http_request_body - default: |- - {} + default: + limit: 1000 + - name: http_client_timeout + default: "60s" - name: interval - default: 0 - - name: json_objects_array - default: "response.Attribute" - - name: pagination - default: |- - { - "enabled": false, - "extra_body_content": {}, - "id_field": "", - "req_field": "", - "url": "" - } + default: "60s" - name: url - default: "" - name: ssl - default: |- - {} input: config/input.yml ingest_pipeline: ingest/pipeline.json diff --git a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log index 3096e9ceaa7b..1d6ba2edb696 100644 --- a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log +++ b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log @@ -1,5 +1,5 @@ {"id":"1","event_id":"1","object_id":"0","object_relation":null,"category":"Network activity","type":"ip-dst","to_ids":false,"uuid":"5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"98.235.162.24","Event":{"org_id":"1","distribution":"3","id":"1","info":"Tor exit nodes feed","orgc_id":"2","uuid":"58dcfe62-ed84-4e5e-b293-4991950d210f"}} {"id":"2","event_id":"2","object_id":"0","object_relation":null,"category":"Payload delivery","type":"md5","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"89357a1b2e32f2b9bddff94b8136810b","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81"}} -{"id":"3","event_id":"3","object_id":"0","object_relation":null,"category":"Payload delivery","type":"filename","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81"}} +{"id":"3","event_id":"3","object_id":"0","object_relation":null,"category":"Payload delivery","type":"filename","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b82","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b82"}} {"id":"4","event_id":"4","object_id":"0","object_relation":null,"category":"Bad Domain","type":"domain","to_ids":true,"uuid":"563b3ea6-b26c-401f-a68b-4d84950d210b","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"4","distribution":"3","id":"4","info":"OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman","orgc_id":"2","uuid":"563b3ea6-b26c-401f-a68b-4d84950d210b"}} {"Galaxy":[],"ShadowAttribute":[],"Tag":[{"colour":"#eb2323","exportable":true,"hide_tag":false,"id":"966","local":0,"name":"critical-ioc-quarantine","numerical_value":null,"user_id":"1"}],"category":"Network activity","comment":"","deleted":false,"disable_correlation":false,"distribution":"5","event_id":"693","first_seen":null,"id":"170814","last_seen":null,"object_id":"0","object_relation":null,"sharing_group_id":"0","timestamp":"1602078120","to_ids":false,"type":"url","uuid":"83078f8c-3d38-4fec-87a1-8be22a0be0a6","value":"endgame.hungmnguyen.us"} diff --git a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json index 7a1d9f95b409..ba55329aaf8a 100644 --- a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json +++ b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json @@ -15,6 +15,7 @@ "event.id": "5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26", "event.kind": "event", "event.module": "misp", + "event.original": "{\"id\":\"1\",\"event_id\":\"1\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Network activity\",\"type\":\"ip-dst\",\"to_ids\":false,\"uuid\":\"5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"98.235.162.24\",\"Event\":{\"org_id\":\"1\",\"distribution\":\"3\",\"id\":\"1\",\"info\":\"Tor exit nodes feed\",\"orgc_id\":\"2\",\"uuid\":\"58dcfe62-ed84-4e5e-b293-4991950d210f\"}}", "event.type": "indicator", "fileset.name": "threat", "input.type": "log", @@ -39,6 +40,7 @@ "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81", "event.kind": "event", "event.module": "misp", + "event.original": "{\"id\":\"2\",\"event_id\":\"2\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Payload delivery\",\"type\":\"md5\",\"to_ids\":true,\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b81\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"89357a1b2e32f2b9bddff94b8136810b\",\"Event\":{\"org_id\":\"1\",\"distribution\":\"3\",\"id\":\"1\",\"info\":\"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass\",\"orgc_id\":\"2\",\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b81\"}}", "event.type": "indicator", "file.hash.md5": "89357a1b2e32f2b9bddff94b8136810b", "fileset.name": "threat", @@ -61,9 +63,10 @@ "@timestamp": "2017-03-30T12:54:26.000Z", "event.category": "threat-intel", "event.dataset": "misp.threat", - "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81", + "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b82", "event.kind": "event", "event.module": "misp", + "event.original": "{\"id\":\"3\",\"event_id\":\"3\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Payload delivery\",\"type\":\"filename\",\"to_ids\":true,\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b82\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\",\"Event\":{\"org_id\":\"1\",\"distribution\":\"3\",\"id\":\"1\",\"info\":\"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass\",\"orgc_id\":\"2\",\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b82\"}}", "event.type": "indicator", "file.path": "f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de", "fileset.name": "threat", @@ -74,12 +77,12 @@ "misp.threat_indicator.attack_pattern_kql": "file.path: \"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\"", "misp.threat_indicator.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass", "misp.threat_indicator.feed": "misp", - "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81", + "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b82", "misp.threat_indicator.type": "filename", "rule.category": "Payload delivery", "rule.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass", "rule.id": "1", - "rule.uuid": "5d159be2-d4b4-4d97-9e14-406a02de0b81", + "rule.uuid": "5d159be2-d4b4-4d97-9e14-406a02de0b82", "service.type": "misp" }, { @@ -89,6 +92,7 @@ "event.id": "563b3ea6-b26c-401f-a68b-4d84950d210b", "event.kind": "event", "event.module": "misp", + "event.original": "{\"id\":\"4\",\"event_id\":\"4\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Bad Domain\",\"type\":\"domain\",\"to_ids\":true,\"uuid\":\"563b3ea6-b26c-401f-a68b-4d84950d210b\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\",\"Event\":{\"org_id\":\"4\",\"distribution\":\"3\",\"id\":\"4\",\"info\":\"OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman\",\"orgc_id\":\"2\",\"uuid\":\"563b3ea6-b26c-401f-a68b-4d84950d210b\"}}", "event.type": "indicator", "fileset.name": "threat", "input.type": "log", @@ -113,6 +117,7 @@ "event.id": "83078f8c-3d38-4fec-87a1-8be22a0be0a6", "event.kind": "event", "event.module": "misp", + "event.original": "{\"Galaxy\":[],\"ShadowAttribute\":[],\"Tag\":[{\"colour\":\"#eb2323\",\"exportable\":true,\"hide_tag\":false,\"id\":\"966\",\"local\":0,\"name\":\"critical-ioc-quarantine\",\"numerical_value\":null,\"user_id\":\"1\"}],\"category\":\"Network activity\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"693\",\"first_seen\":null,\"id\":\"170814\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1602078120\",\"to_ids\":false,\"type\":\"url\",\"uuid\":\"83078f8c-3d38-4fec-87a1-8be22a0be0a6\",\"value\":\"endgame.hungmnguyen.us\"}", "event.type": "indicator", "fileset.name": "threat", "input.type": "log", diff --git a/x-pack/filebeat/modules.d/misp.yml.disabled b/x-pack/filebeat/modules.d/misp.yml.disabled index 0299a64eb6d0..25cbaf6b1b84 100644 --- a/x-pack/filebeat/modules.d/misp.yml.disabled +++ b/x-pack/filebeat/modules.d/misp.yml.disabled @@ -8,13 +8,10 @@ #var.api_key # Array object in MISP response - #var.json_objects_array + #var.http_request_body.limit: 1000 # URL of the MISP REST API #var.url - + # You can also pass SSL options. For example: - #var.ssl: |- - # { - # verification_mode: none - # } + #var.ssl.verification_mode: none