From 37808d60b16550246eead959a508141aa46716a1 Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Sat, 29 Apr 2023 17:00:32 +0200 Subject: [PATCH 01/29] Update first version of query schema --- .schemas/queries.json | 233 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 .schemas/queries.json diff --git a/.schemas/queries.json b/.schemas/queries.json new file mode 100644 index 000000000..04c73a8d4 --- /dev/null +++ b/.schemas/queries.json @@ -0,0 +1,233 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "metadata": { + "$ref": "#/$defs/metadata" + }, + "defaults": { + "type": "object", + "properties": { + "metadata": { + "type": "object", + "properties": { + "data_source": { + "type": "string" + }, + "pivot": { + "type": "object", + "patternProperties": { + ".*": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "top": { + "type": "integer" + } + } + }, + "parameters": { + "anyOf": [ + { + "type": "object", + "patternProperties": { + ".*": { + "$ref": "#/$defs/parameter" + } + } + }, + true + ] + } + } + }, + "sources": { + "type": "object", + "patternProperties": { + ".*": { + "$ref": "#/$defs/query" + } + } + } + }, + "$defs": { + "description": { + "type": "string", + "minLength": 10, + "maxLength": 256 + }, + "metadata": { + "type": "object", + "properties": { + "version": { + "type": "integer" + }, + "description": { + "$ref": "#/$defs/description" + }, + "data_environments": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "AzureSecurityCenter", + "AzureSentinel", + "Cybereason", + "Elastic", + "Kusto", + "LocalData", + "LogAnalytics", + "M365D", + "MDATP", + "MDE", + "Mordor", + "MSGraph", + "MSSentinel", + "OSQueryLogs", + "OTRF", + "ResourceGraph", + "SecurityGraph", + "Splunk", + "Sumologic" + ] + } + }, + "data_families": { + "type": "array", + "items": { + "type": "string" + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "aliases": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false, + "required": [ + "version", + "description", + "data_environments", + "data_families" + ] + }, + "parameter": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/description" + }, + "type": { + "type": "string", + "enum": [ + "str", + "datetime", + "int", + "float", + "list" + ] + }, + "default": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "integer" + } + ] + }, + "additionalProperties": false, + "required": [ + "description", + "type" + ] + } + }, + "query": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/description" + }, + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/metadata" + }, + true + ] + }, + "parameters": { + "anyOf": [ + { + "$ref": "#/$defs/parameter" + }, + true + ] + }, + "args": { + "type": "object", + "properties": { + "query": { + "type": "string" + }, + "uri": { + "type": "string" + } + }, + "required": [ + "query" + ] + }, + "query_macros": { + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/description" + }, + "value": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "description", + "value" + ] + } + } + } + }, + "additionalProperties": false, + "required": [ + "description", + "args" + ] + } + } +} \ No newline at end of file From 0b7c7dd13e5ab594825976cce4ff608504054a16 Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Sat, 29 Apr 2023 17:03:59 +0200 Subject: [PATCH 02/29] Adding JSON Schema library to dependencies --- requirements-all.txt | 1 + requirements-dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-all.txt b/requirements-all.txt index a82655fd5..1224fbc87 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -24,6 +24,7 @@ html5lib ipython >= 7.1.1; python_version < "3.8" ipython >= 7.23.1; python_version >= "3.8" ipywidgets>=7.4.2, <8.0.0 +jsonschema>=4.17.3 keyring>=13.2.1 KqlmagicCustom[jupyter-basic,auth_code_clipboard]>=0.1.114.post22 KqlmagicCustom[jupyter-extended]>=0.1.114.post22 diff --git a/requirements-dev.txt b/requirements-dev.txt index 79bc60cd9..28c03badd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,6 +8,7 @@ docutils<0.20.0 filelock>=3.0.0 flake8>=3.8.4 isort>=5.10.1 +jsonschema>=4.17.3 markdown>=3.3.4 mccabe>=0.6.1 mypy>=0.812 From 6f9f917e3b68cd965b7d2426fa89aee31445fdbf Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Sat, 29 Apr 2023 17:04:20 +0200 Subject: [PATCH 03/29] Add tests to validate the correctness of existing queries --- tests/common/test_pkg_config.py | 26 ++++++++++++++++++++++---- tests/unit_test_lib.py | 8 ++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index 7d28f8d88..31e2f31cb 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -9,6 +9,7 @@ from pathlib import Path import httpx +from jsonschema import Draft7Validator import pytest import pytest_check as check import yaml @@ -16,13 +17,24 @@ from msticpy.common import pkg_config from msticpy.context.geoip import GeoLiteLookup, IPStackLookup -from ..unit_test_lib import custom_mp_config, get_test_data_path +from ..unit_test_lib import custom_mp_config, get_test_data_path, get_queries_schema _TEST_DATA = get_test_data_path() +_QUERIES_SCHEMA = get_queries_schema() # pylint: disable=protected-access +def validate_queries_file_structure(query_file: Path): + """Test if query files have a valid structure.""" + with query_file.open(mode="r", encoding="utf-8") as queries: + queries_yaml = yaml.safe_load(queries) + check.is_true( + Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), + msg=f"File {query_file} is not a valid query file", + ) + + def test_load_default(): """Test load default settings.""" settings = pkg_config._settings @@ -31,10 +43,16 @@ def test_load_default(): check.equal(1, len(settings["QueryDefinitions"]["Default"])) for path in settings["QueryDefinitions"]["Default"]: check.is_true(type(path), str) - path = f"data/{path}" - check.is_true( - Path(pkg_config.__file__).resolve().parent.parent.joinpath(path).is_dir() + path = ( + Path(pkg_config.__file__).resolve().parent.parent.joinpath(f"data/{path}") ) + check.is_true(path.is_dir()) + for query_file in path.rglob("*.yaml"): + validate_queries_file_structure(query_file) + for path in settings["QueryDefinitions"].get("Custom", []): + check.is_true(type(path), str) + for query_file in path.rglob("*.yaml"): + validate_queries_file_structure(query_file) def test_custom_config(): diff --git a/tests/unit_test_lib.py b/tests/unit_test_lib.py index 48d841c66..4f5d16ce1 100644 --- a/tests/unit_test_lib.py +++ b/tests/unit_test_lib.py @@ -13,6 +13,7 @@ import nbformat from filelock import FileLock +import yaml from nbconvert.preprocessors import CellExecutionError, ExecutePreprocessor from msticpy.common import pkg_config @@ -25,6 +26,13 @@ def get_test_data_path(): return Path(__file__).parent.joinpath("testdata") +def get_queries_schema(): + """Get queries schema.""" + queries_schema_path = Path(__file__).parent.parent.joinpath(".schemas").joinpath("queries.json") + with queries_schema_path.open(mode="r", encoding="utf-8") as queries_schema: + return yaml.safe_load(queries_schema) + + TEST_DATA_PATH = str(get_test_data_path()) From 5f07ad651009d6a78a6637cf971731665de4c3c0 Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Sat, 29 Apr 2023 17:07:00 +0200 Subject: [PATCH 04/29] Fix typos & missing description in queries --- msticpy/data/queries/mssentinel/kql_sent_azure.yaml | 4 ++-- msticpy/data/queries/mssentinel/kql_sent_net.yaml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/msticpy/data/queries/mssentinel/kql_sent_azure.yaml b/msticpy/data/queries/mssentinel/kql_sent_azure.yaml index fc3f916d2..cb66f376d 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_azure.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_azure.yaml @@ -170,7 +170,7 @@ sources: type: str default: SigninLogs list_storage_ops_for_ip: - description: + description: List storage operations for IP metadata: pivot: short_name: storage_ops @@ -203,7 +203,7 @@ sources: description: Client IP Address type: str list_storage_ops_for_hash: - description: + description: List storage operations for Hash args: query: ' union diff --git a/msticpy/data/queries/mssentinel/kql_sent_net.yaml b/msticpy/data/queries/mssentinel/kql_sent_net.yaml index e1dd46e0c..146ec4722 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_net.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_net.yaml @@ -252,7 +252,7 @@ sources: description: URL to search for type: str host_network_connections_csl: - descritpion: List network connections to and from a host using CommonSecurityLog + description: List network connections to and from a host using CommonSecurityLog metadata: pivot: short_name: host_connections_csl @@ -277,7 +277,7 @@ sources: type: str default: "" ip_network_connections_csl: - descritpion: List network connections to and from an IP address using CommonSecurityLog + description: List network connections to and from an IP address using CommonSecurityLog metadata: pivot: short_name: ip_connections_csl @@ -302,7 +302,7 @@ sources: type: str default: "" all_network_connections_csl: - descritpion: List network connections to and from an IP address using CommonSecurityLog + description: List network connections to and from an IP address using CommonSecurityLog args: query: ' {table} @@ -318,7 +318,7 @@ sources: type: str default: "CommonSecurityLog" ips_by_host_csl: - descritpion: All IP addresses associated with a host using CommonSecurityLog + description: All IP addresses associated with a host using CommonSecurityLog metadata: pivot: short_name: ips_csl @@ -344,7 +344,7 @@ sources: type: str default: "" hosts_by_ip_csl: - descritpion: All hosts associated with a IP addresses using CommonSecurityLog + description: All hosts associated with a IP addresses using CommonSecurityLog metadata: pivot: short_name: hosts_csl From 5e293161b98ce622f26bce40f2762d5ed6228670 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 29 Apr 2023 22:41:56 +0200 Subject: [PATCH 05/29] Convert path to Path type --- tests/common/test_pkg_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index 31e2f31cb..bbf420d46 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -51,7 +51,9 @@ def test_load_default(): validate_queries_file_structure(query_file) for path in settings["QueryDefinitions"].get("Custom", []): check.is_true(type(path), str) - for query_file in path.rglob("*.yaml"): + path = Path(path) + check.is_true(path.is_dir()) + for query_file in Path(path).resolve().rglob("*.yaml"): validate_queries_file_structure(query_file) From c3c6c85c9097808e9679e8dc44f3d038f6890780 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 29 Apr 2023 23:21:07 +0200 Subject: [PATCH 06/29] If path is not a dir, use relative path from config file --- tests/common/test_pkg_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index bbf420d46..4901863e6 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -52,6 +52,8 @@ def test_load_default(): for path in settings["QueryDefinitions"].get("Custom", []): check.is_true(type(path), str) path = Path(path) + if not path.is_absolute(): + path = Path(pkg_config.__file__).resolve().joinpath(path) check.is_true(path.is_dir()) for query_file in Path(path).resolve().rglob("*.yaml"): validate_queries_file_structure(query_file) From 599b9b99e2f77d7655ba161b2b4c9e636c9d0ac0 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 30 Apr 2023 10:32:49 +0200 Subject: [PATCH 07/29] Update validation method to handle expected fails --- tests/common/test_pkg_config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index 4901863e6..46f1602ac 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -25,14 +25,20 @@ # pylint: disable=protected-access -def validate_queries_file_structure(query_file: Path): +def validate_queries_file_structure(query_file: Path, expected: bool = True): """Test if query files have a valid structure.""" with query_file.open(mode="r", encoding="utf-8") as queries: queries_yaml = yaml.safe_load(queries) - check.is_true( - Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), - msg=f"File {query_file} is not a valid query file", - ) + if expected: + check.is_true( + Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), + msg=f"File {query_file} is not a valid query file", + ) + else: + check.is_false( + Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), + msg=f"File {query_file} was expected to be invalid but is a valid query file", + ) def test_load_default(): From e4e7da74dc79927d11071a4d1ec1e26e7552687c Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 30 Apr 2023 10:34:40 +0200 Subject: [PATCH 08/29] Fix path resolution for custom queries Handle expected failures --- tests/common/test_pkg_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index 46f1602ac..a7de5b3bc 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -59,10 +59,12 @@ def test_load_default(): check.is_true(type(path), str) path = Path(path) if not path.is_absolute(): - path = Path(pkg_config.__file__).resolve().joinpath(path) + path = Path(__file__).resolve().parent.parent.joinpath(path) check.is_true(path.is_dir()) for query_file in Path(path).resolve().rglob("*.yaml"): - validate_queries_file_structure(query_file) + validate_queries_file_structure( + query_file, expected=not ("fail" in query_file.name) + ) def test_custom_config(): From e30ccd85bac4e0a671e73e6c62b170d9e147a2a7 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 30 Apr 2023 10:35:27 +0200 Subject: [PATCH 09/29] Add missing keys Ensure parameters are always defined --- .schemas/queries.json | 104 +++++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/.schemas/queries.json b/.schemas/queries.json index 04c73a8d4..b3e1eabf5 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -14,6 +14,12 @@ "data_source": { "type": "string" }, + "data_families": { + "type": "array", + "items": { + "type": "string" + } + }, "pivot": { "type": "object", "patternProperties": { @@ -31,17 +37,7 @@ } }, "parameters": { - "anyOf": [ - { - "type": "object", - "patternProperties": { - ".*": { - "$ref": "#/$defs/parameter" - } - } - }, - true - ] + "$ref": "#/$defs/parameter" } } }, @@ -57,7 +53,7 @@ "$defs": { "description": { "type": "string", - "minLength": 10, + "minLength": 5, "maxLength": 256 }, "metadata": { @@ -120,6 +116,9 @@ "type": "string" } ] + }, + "cluster": { + "type": "string" } }, "additionalProperties": false, @@ -132,38 +131,56 @@ }, "parameter": { "type": "object", - "properties": { - "description": { - "$ref": "#/$defs/description" - }, - "type": { - "type": "string", - "enum": [ - "str", - "datetime", - "int", - "float", - "list" - ] - }, - "default": { - "oneOf": [ - { - "type": "string" + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/description" }, - { - "type": "array" + "type": { + "type": "string", + "enum": [ + "str", + "datetime", + "int", + "float", + "list" + ] }, - { - "type": "integer" + "default": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "integer" + } + ] + }, + "aliases": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] } + }, + "additionalProperties": false, + "required": [ + "description", + "type" ] - }, - "additionalProperties": false, - "required": [ - "description", - "type" - ] + } } }, "query": { @@ -181,12 +198,7 @@ ] }, "parameters": { - "anyOf": [ - { - "$ref": "#/$defs/parameter" - }, - true - ] + "$ref": "#/$defs/parameter" }, "args": { "type": "object", From 9c9c1c2a170c0bbfaec3c74b152b54174a6f242f Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 30 Apr 2023 10:35:54 +0200 Subject: [PATCH 10/29] Fix incorrect type names --- msticpy/data/queries/cybereason/cybereason_processes.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msticpy/data/queries/cybereason/cybereason_processes.yaml b/msticpy/data/queries/cybereason/cybereason_processes.yaml index 001f5b45c..849780484 100644 --- a/msticpy/data/queries/cybereason/cybereason_processes.yaml +++ b/msticpy/data/queries/cybereason/cybereason_processes.yaml @@ -78,7 +78,7 @@ sources: parameters: command: description: Command to search for - type: string + type: str find_process_by_suspicions: description: Search for process with a specific suspicion metadata: @@ -108,7 +108,7 @@ sources: parameters: suspicion: description: Suspicion that the process should have - type: string + type: str find_process_by_pid: description: Search for a process by pid and hostname metadata: From 5d3808dc9f1aed1aede12e470192652419c7198d Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 30 Apr 2023 10:51:50 +0200 Subject: [PATCH 11/29] Remove empty parameters from query files --- .../data/queries/localdata/local_data.yaml | 10 --------- .../data/queries/m365d/kql_mdatp_alerts.yaml | 4 +--- .../data/queries/m365d/kql_mdatp_hunting.yaml | 21 +------------------ .../data/queries/m365d/kql_mdatp_network.yaml | 1 - .../data/queries/mde/kql_mdatp_alerts.yaml | 1 - .../data/queries/mde/kql_mdatp_hunting.yaml | 21 +------------------ .../data/queries/mde/kql_mdatp_network.yaml | 1 - .../data/queries/msgraph/graph_alerts.yaml | 1 - .../mssentinel/kql_sent_azuresentinel.yaml | 1 - .../queries/mssentinel/kql_sent_lxauditd.yaml | 1 - .../kql_sent_lxsyslog_activity.yaml | 5 ----- .../mssentinel/kql_sent_lxsyslog_logon.yaml | 1 - .../mssentinel/kql_sent_threatintel.yaml | 1 - .../queries/mssentinel/kql_sent_winevent.yaml | 1 - .../resourcegraph/resource_graph_queries.yaml | 4 ---- .../data/queries/splunk/splunk_queries.yaml | 7 +------ .../queries/sumologic/sumologic_queries.yaml | 1 - tests/testdata/data_q_hierarchy.yaml | 2 -- tests/testdata/data_q_success.yaml | 2 -- .../localdata/test_localdata_queries.yaml | 3 --- 20 files changed, 4 insertions(+), 85 deletions(-) diff --git a/msticpy/data/queries/localdata/local_data.yaml b/msticpy/data/queries/localdata/local_data.yaml index de8da3a61..be707db3e 100644 --- a/msticpy/data/queries/localdata/local_data.yaml +++ b/msticpy/data/queries/localdata/local_data.yaml @@ -7,7 +7,6 @@ metadata: defaults: metadata: data_source: 'security_alert' - parameters: sources: list_alerts: description: Retrieves list of alerts @@ -15,60 +14,51 @@ sources: data_families: [SecurityAlert] args: query: alerts_list.pkl - parameters: list_host_processes: description: List processes on host metadata: data_families: [WindowsSecurity] args: query: processes_on_host.pkl - parameters: list_host_logons: description: List logons on host metadata: data_families: [WindowsSecurity] args: query: host_logons.pkl - parameters: list_host_logon_failures: description: List logon failures on host metadata: data_families: [WindowsSecurity] args: query: failed_logons.pkl - parameters: list_host_events: description: List events failures on host metadata: data_families: [WindowsSecurity] args: query: all_events_df.pkl - parameters: get_process_tree: description: Get process tree for a process metadata: data_families: [WindowsSecurity] args: query: process_tree.pkl - parameters: list_azure_network_flows_by_ip: description: List Azure Network flows by IP address metadata: data_families: [Network] args: query: az_net_comms_df.pkl - parameters: list_azure_network_flows_by_host: description: List Azure Network flows by host name metadata: data_families: [Network] args: query: az_net_comms_df.pkl - parameters: list_all_signins_geo: description: List all Azure AD logon events metadata: data_families: [Azure] args: query: aad_logons.pkl - parameters: \ No newline at end of file diff --git a/msticpy/data/queries/m365d/kql_mdatp_alerts.yaml b/msticpy/data/queries/m365d/kql_mdatp_alerts.yaml index c9d9a0c4b..1fdf7c1f9 100644 --- a/msticpy/data/queries/m365d/kql_mdatp_alerts.yaml +++ b/msticpy/data/queries/m365d/kql_mdatp_alerts.yaml @@ -33,7 +33,6 @@ sources: | where Timestamp <= datetime({end}) {add_query_items}" uri: None - parameters: list_alerts_with_evidence: description: Retrieves list of alerts with their evidence metadata: @@ -48,7 +47,6 @@ sources: ) {add_query_items}" uri: None - parameters: host_alerts: description: Lists alerts by for a specified hostname metadata: @@ -252,4 +250,4 @@ sources: parameters: app_name: description: The name of the application - type: str \ No newline at end of file + type: str diff --git a/msticpy/data/queries/m365d/kql_mdatp_hunting.yaml b/msticpy/data/queries/m365d/kql_mdatp_hunting.yaml index 4efd6e1b1..12c211a51 100644 --- a/msticpy/data/queries/m365d/kql_mdatp_hunting.yaml +++ b/msticpy/data/queries/m365d/kql_mdatp_hunting.yaml @@ -61,7 +61,6 @@ sources: by bin({time_column}, 1tick), FileName, FileOriginUrl, ClickUrl, SHA1, DeviceName, DeviceId {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Doc%20attachment%20with%20link%20to%20download.txt" - parameters: dropbox_link: description: Looks for user content downloads from dropbox that originate from a link/redirect from a 3rd party site. metadata: @@ -76,7 +75,6 @@ sources: | project FileOriginReferrerUrl, FileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Dropbox%20downloads%20linked%20from%20other%20site.txt" - parameters: email_smartscreen: description: Look for links opened from outlook.exe, followed by a browser download and then a SmartScreen app warning metadata: @@ -112,7 +110,6 @@ sources: | distinct * {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Email%20link%20%2B%20download%20%2B%20SmartScreen%20warning.txt" - parameters: email_link: description: Look for links opened from mail apps – if a detection occurred right afterwards metadata: @@ -146,7 +143,6 @@ sources: | summarize FirstDetectedActivity=min(FirstDetectedActivity), AlertTitles=makeset(Title) by OpenedLink, InitiatingProcessFileName, {time_column}=bin({time_column}, 1tick), DeviceName, DeviceId, WasOutlookSafeLink {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Open%20email%20link.txt" - parameters: av_sites: description: Pivot from downloads detected by Windows Defender Antivirus to other files downloaded from the same sites metadata: @@ -190,7 +186,6 @@ sources: | order by MachineCount desc {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Pivot%20from%20detections%20to%20related%20downloads.txt" - parameters: tor: description: Looks for Tor client, or for a common Tor plugin called Meek. metadata: @@ -204,7 +199,6 @@ sources: | order by MachineCount desc {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Command%20and%20Control/Tor.txt" - parameters: network_scans: description: Looking for high volume queries against a given RemoteIP, per ComputerName, RemotePort and Process metadata: @@ -219,7 +213,6 @@ sources: | where RemotePortCount > remotePortCountThreshold {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Command%20and%20Control/Tor.txt" - parameters: user_enumeration: description: The query finds attempts to list users or groups using Net commands metadata: @@ -234,7 +227,6 @@ sources: | sort by AccountName, Target {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Discovery/Enumeration%20of%20users%20%26%20groups%20for%20lateral%20movement.txt" - parameters: smb_discovery: description: Query for processes that accessed more than 10 IP addresses over port 445 (SMB) - possibly scanning for network shares. metadata: @@ -248,7 +240,6 @@ sources: | where RemoteIPCount > 10 {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Discovery/SMB%20shares%20discovery.txt" - parameters: b64_pe: description: Finding base64 encoded PE files header seen in the command line parameters metadata: @@ -260,7 +251,6 @@ sources: | top 1000 by {time_column} {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/Base64encodePEFile.txt" - parameters: malware_recycle: description: Finding attackers hiding malware in the recycle bin. metadata: @@ -273,7 +263,6 @@ sources: | project {time_column}, DeviceName, ProcessCommandLine, InitiatingProcessFileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/Malware_In_recyclebin.txt" - parameters: powershell_downloads: description: Finds PowerShell execution events that could involve a download. metadata: @@ -294,7 +283,6 @@ sources: | top 100 by {time_column} {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/PowerShell%20downloads.txt" - parameters: uncommon_powershell: description: Find which uncommon Powershell Cmdlets were executed on that machine in a certain time period. metadata: @@ -336,7 +324,6 @@ sources: | summarize FirstEvent=min({time_column}), LastEvent=max({time_column}) by DeviceName, ProcessCommandLine, FileName, InitiatingProcessFileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Electron-CVE-2018-1000006.txt" - parameters: cve_2018_4878: description: This query checks for specific processes and domain TLD used in the CVE-2018-4878 metadata: @@ -350,7 +337,6 @@ sources: | top 100 by {time_column} {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Flash-CVE-2018-4848.txt" - parameters: cve_2018_1111: description: Looks for CVE-2018-1111 exploitation metadata: @@ -364,7 +350,6 @@ sources: | project {time_column}, DeviceName , FileName, ProcessCommandLine, InitiatingProcessCommandLine {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Linux-DynoRoot-CVE-2018-1111.txt" - parameters: brute_force: description: Look for public IP addresses that failed to logon to a computer multiple times, using multiple accounts, and eventually succeeded. metadata: @@ -386,7 +371,6 @@ sources: | where Failed > 10 and Successful > 0 and FailedAccountsCount > 2 and SuccessfulAccountsCount == 1 {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Lateral%20Movement/Account%20brute%20force.txt" - parameters: service_account_powershell: description: Service Accounts Performing Remote PowerShell metadata: @@ -432,7 +416,6 @@ sources: | where min_Timestamp > ago(1d) {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Lateral%20Movement/ServiceAccountsPerformingRemotePS.txt" - parameters: accessibility_persistence: description: This query looks for persistence or privilege escalation done using Windows Accessibility features. metadata: @@ -478,7 +461,6 @@ sources: | union kind=outer executedProcessIsPowershellOrCmd {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Persistence/Accessibility%20Features.txt" - parameters: smartscreen_ignored: description: Query for SmartScreen URL blocks, where the user has decided to run the malware nontheless. metadata: @@ -509,5 +491,4 @@ sources: | where DownloadTime - {time_column} between (0min .. 2min) | project-away DeviceName1 {add_query_items}' - uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Protection%20events/SmartScreen%20URL%20block%20ignored%20by%20user.txt" - parameters: \ No newline at end of file + uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Protection%20events/SmartScreen%20URL%20block%20ignored%20by%20user.txt" \ No newline at end of file diff --git a/msticpy/data/queries/m365d/kql_mdatp_network.yaml b/msticpy/data/queries/m365d/kql_mdatp_network.yaml index 31f8b18e2..ce0e74fc3 100644 --- a/msticpy/data/queries/m365d/kql_mdatp_network.yaml +++ b/msticpy/data/queries/m365d/kql_mdatp_network.yaml @@ -37,7 +37,6 @@ sources: | where {time_column} <= datetime({end}) {add_query_items}" uri: None - parameters: host_connections: description: Lists connections by for a specified hostname metadata: diff --git a/msticpy/data/queries/mde/kql_mdatp_alerts.yaml b/msticpy/data/queries/mde/kql_mdatp_alerts.yaml index bd2478585..1c93fff72 100644 --- a/msticpy/data/queries/mde/kql_mdatp_alerts.yaml +++ b/msticpy/data/queries/mde/kql_mdatp_alerts.yaml @@ -33,7 +33,6 @@ sources: | where Timestamp <= datetime({end}) {add_query_items}" uri: None - parameters: host_alerts: description: Lists alerts by for a specified hostname metadata: diff --git a/msticpy/data/queries/mde/kql_mdatp_hunting.yaml b/msticpy/data/queries/mde/kql_mdatp_hunting.yaml index 7e0af9743..7b6f4a806 100644 --- a/msticpy/data/queries/mde/kql_mdatp_hunting.yaml +++ b/msticpy/data/queries/mde/kql_mdatp_hunting.yaml @@ -57,7 +57,6 @@ sources: by bin(Timestamp, 1tick), FileName, FileOriginUrl, ClickUrl, SHA1, DeviceName, DeviceId {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Doc%20attachment%20with%20link%20to%20download.txt" - parameters: dropbox_link: description: Looks for user content downloads from dropbox that originate from a link/redirect from a 3rd party site. metadata: @@ -72,7 +71,6 @@ sources: | project FileOriginReferrerUrl, FileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Dropbox%20downloads%20linked%20from%20other%20site.txt" - parameters: email_smartscreen: description: Look for links opened from outlook.exe, followed by a browser download and then a SmartScreen app warning metadata: @@ -108,7 +106,6 @@ sources: | distinct * {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Email%20link%20%2B%20download%20%2B%20SmartScreen%20warning.txt" - parameters: email_link: description: Look for links opened from mail apps – if a detection occurred right afterwards metadata: @@ -142,7 +139,6 @@ sources: | summarize FirstDetectedActivity=min(FirstDetectedActivity), AlertTitles=makeset(Title) by OpenedLink, InitiatingProcessFileName, Timestamp=bin(Timestamp, 1tick), DeviceName, DeviceId, WasOutlookSafeLink {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Open%20email%20link.txt" - parameters: av_sites: description: Pivot from downloads detected by Windows Defender Antivirus to other files downloaded from the same sites metadata: @@ -186,7 +182,6 @@ sources: | order by MachineCount desc {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Pivot%20from%20detections%20to%20related%20downloads.txt" - parameters: tor: description: Looks for Tor client, or for a common Tor plugin called Meek. metadata: @@ -200,7 +195,6 @@ sources: | order by MachineCount desc {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Command%20and%20Control/Tor.txt" - parameters: network_scans: description: Looking for high volume queries against a given RemoteIP, per ComputerName, RemotePort and Process metadata: @@ -215,7 +209,6 @@ sources: | where RemotePortCount > remotePortCountThreshold {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Command%20and%20Control/Tor.txt" - parameters: user_enumeration: description: The query finds attempts to list users or groups using Net commands metadata: @@ -230,7 +223,6 @@ sources: | sort by AccountName, Target {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Discovery/Enumeration%20of%20users%20%26%20groups%20for%20lateral%20movement.txt" - parameters: smb_discovery: description: Query for processes that accessed more than 10 IP addresses over port 445 (SMB) - possibly scanning for network shares. metadata: @@ -244,7 +236,6 @@ sources: | where RemoteIPCount > 10 {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Discovery/SMB%20shares%20discovery.txt" - parameters: b64_pe: description: Finding base64 encoded PE files header seen in the command line parameters metadata: @@ -256,7 +247,6 @@ sources: | top 1000 by Timestamp {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/Base64encodePEFile.txt" - parameters: malware_recycle: description: Finding attackers hiding malware in the recycle bin. metadata: @@ -269,7 +259,6 @@ sources: | project Timestamp, DeviceName, ProcessCommandLine, InitiatingProcessFileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/Malware_In_recyclebin.txt" - parameters: powershell_downloads: description: Finds PowerShell execution events that could involve a download. metadata: @@ -290,7 +279,6 @@ sources: | top 100 by Timestamp {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/PowerShell%20downloads.txt" - parameters: uncommon_powershell: description: Find which uncommon Powershell Cmdlets were executed on that machine in a certain time period. metadata: @@ -332,7 +320,6 @@ sources: | summarize FirstEvent=min(Timestamp), LastEvent=max(Timestamp) by DeviceName, ProcessCommandLine, FileName, InitiatingProcessFileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Electron-CVE-2018-1000006.txt" - parameters: cve_2018_4878: description: This query checks for specific processes and domain TLD used in the CVE-2018-4878 metadata: @@ -346,7 +333,6 @@ sources: | top 100 by Timestamp {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Flash-CVE-2018-4848.txt" - parameters: cve_2018_1111: description: Looks for CVE-2018-1111 exploitation metadata: @@ -360,7 +346,6 @@ sources: | project Timestamp, DeviceName , FileName, ProcessCommandLine, InitiatingProcessCommandLine {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Linux-DynoRoot-CVE-2018-1111.txt" - parameters: brute_force: description: Look for public IP addresses that failed to logon to a computer multiple times, using multiple accounts, and eventually succeeded. metadata: @@ -382,7 +367,6 @@ sources: | where Failed > 10 and Successful > 0 and FailedAccountsCount > 2 and SuccessfulAccountsCount == 1 {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Lateral%20Movement/Account%20brute%20force.txt" - parameters: service_account_powershell: description: Service Accounts Performing Remote PowerShell metadata: @@ -428,7 +412,6 @@ sources: | where min_Timestamp > ago(1d) {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Lateral%20Movement/ServiceAccountsPerformingRemotePS.txt" - parameters: accessibility_persistence: description: This query looks for persistence or privilege escalation done using Windows Accessibility features. metadata: @@ -474,7 +457,6 @@ sources: | union kind=outer executedProcessIsPowershellOrCmd {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Persistence/Accessibility%20Features.txt" - parameters: smartscreen_ignored: description: Query for SmartScreen URL blocks, where the user has decided to run the malware nontheless. metadata: @@ -505,5 +487,4 @@ sources: | where DownloadTime - Timestamp between (0min .. 2min) | project-away DeviceName1 {add_query_items}' - uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Protection%20events/SmartScreen%20URL%20block%20ignored%20by%20user.txt" - parameters: \ No newline at end of file + uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Protection%20events/SmartScreen%20URL%20block%20ignored%20by%20user.txt" \ No newline at end of file diff --git a/msticpy/data/queries/mde/kql_mdatp_network.yaml b/msticpy/data/queries/mde/kql_mdatp_network.yaml index 31402ff83..ff7693d1f 100644 --- a/msticpy/data/queries/mde/kql_mdatp_network.yaml +++ b/msticpy/data/queries/mde/kql_mdatp_network.yaml @@ -33,7 +33,6 @@ sources: | where Timestamp <= datetime({end}) {add_query_items}" uri: None - parameters: host_connections: description: Lists alerts by for a specified hostname metadata: diff --git a/msticpy/data/queries/msgraph/graph_alerts.yaml b/msticpy/data/queries/msgraph/graph_alerts.yaml index a26a2bbef..62ff93344 100644 --- a/msticpy/data/queries/msgraph/graph_alerts.yaml +++ b/msticpy/data/queries/msgraph/graph_alerts.yaml @@ -39,7 +39,6 @@ sources: query: '{path}?$filter=createdDateTime ge {start} and createdDateTime le {end} {add_query_items}' uri: None - parameters: get_alert: description: Retrieves a single alert by AlertId metadata: diff --git a/msticpy/data/queries/mssentinel/kql_sent_azuresentinel.yaml b/msticpy/data/queries/mssentinel/kql_sent_azuresentinel.yaml index 9784f87a0..164d9ea7f 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_azuresentinel.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_azuresentinel.yaml @@ -43,7 +43,6 @@ sources: | project-away QryResults | where SoftDeleted == false {add_query_items}' - parameters: list_bookmarks_for_entity: description: Retrieves bookmarks for entity string metadata: diff --git a/msticpy/data/queries/mssentinel/kql_sent_lxauditd.yaml b/msticpy/data/queries/mssentinel/kql_sent_lxauditd.yaml index bf3712b47..9b919aa82 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_lxauditd.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_lxauditd.yaml @@ -48,4 +48,3 @@ sources: | extend typed_mssg = pack(mssg_type, mssg_content) | summarize AuditdMessage = makelist(typed_mssg) by TenantId, TimeGenerated, Computer, mssg_id {add_query_items}' - parameters: diff --git a/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_activity.yaml b/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_activity.yaml index b9de423d5..353d44814 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_activity.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_activity.yaml @@ -86,7 +86,6 @@ sources: | where ProcessName == "CRON" or Facility == "cron" | extend CMD=extract("CMD(.*)",1,SyslogMessage), User=extract("for user ([[:alpha:]]*)",1,SyslogMessage), CronUser=extract("^[(]([[:alpha:]]*)",1,SyslogMessage),EditStatus=extract("[A-Z]+ EDIT",0,SyslogMessage) {add_query_items}' - parameters: user_group_activity: description: All user/group additions, deletions, and modifications args: @@ -105,7 +104,6 @@ sources: | extend User=extract("(user: name=|user '')([[:alnum:]]+)",2,SyslogMessage), Group=extract("(group: name=|group '')([[:alnum:]]+)",2,SyslogMessage), UID=extract("UID=([0-9]+)",1,SyslogMessage), GID=extract("GID=([0-9]+)",1,SyslogMessage) | where UserGroupAction != "" {add_query_items}' - parameters: all_syslog: description: Returns all syslog activity for a host args: @@ -116,7 +114,6 @@ sources: | where TimeGenerated <= datetime({end}) | where Computer {host_op} "{host_name}" {add_query_items}' - parameters: summarize_events: description: Returns all syslog activity for a host args: @@ -128,7 +125,6 @@ sources: | where Computer {host_op} "{host_name}" | summarize count() by Facility, SeverityLevel, ProcessName {add_query_items}' - parameters: notable_events: description: Returns all syslog activity for a host args: @@ -140,4 +136,3 @@ sources: | where Computer {host_op} "{host_name}" | where SeverityLevel in ("alert", "crit") {add_query_items}' - parameters: diff --git a/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_logon.yaml b/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_logon.yaml index fed5eabb0..8597a60b4 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_logon.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_lxsyslog_logon.yaml @@ -305,7 +305,6 @@ sources: description: All failed user logon events on any host args: query: *logon_failures - parameters: query_macros: query_condition: description: Null where clause diff --git a/msticpy/data/queries/mssentinel/kql_sent_threatintel.yaml b/msticpy/data/queries/mssentinel/kql_sent_threatintel.yaml index 652599fb2..d103d8805 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_threatintel.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_threatintel.yaml @@ -39,7 +39,6 @@ sources: | summarize arg_max(TimeGenerated, *) by IndicatorId {add_query_items}' uri: None - parameters: list_indicators_by_ip: description: Retrieves list of indicators by IP Address metadata: diff --git a/msticpy/data/queries/mssentinel/kql_sent_winevent.yaml b/msticpy/data/queries/mssentinel/kql_sent_winevent.yaml index 7e3ab9c6e..02554aeb0 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_winevent.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_winevent.yaml @@ -106,7 +106,6 @@ sources: | where TimeGenerated >= datetime({start}) | where TimeGenerated <= datetime({end}) {add_query_items}' - parameters: list_events_by_id: description: Retrieves list of events on a host args: diff --git a/msticpy/data/queries/resourcegraph/resource_graph_queries.yaml b/msticpy/data/queries/resourcegraph/resource_graph_queries.yaml index 7f434ca8d..f7380cde3 100644 --- a/msticpy/data/queries/resourcegraph/resource_graph_queries.yaml +++ b/msticpy/data/queries/resourcegraph/resource_graph_queries.yaml @@ -26,7 +26,6 @@ sources: {table} {add_query_items}' uri: None - parameters: list_resources_by_type: description: Retrieves list of resources by type metadata: @@ -51,7 +50,6 @@ sources: | order by name desc {add_query_items}' uri: None - parameters: list_public_ips: description: Retrieves list of resources with public IP addresses metadata: @@ -62,7 +60,6 @@ sources: | project properties.ipAddress {add_query_items}' uri: None - parameters: list_resources_by_api_version: description: Retrieves list of resources for each API version metadata: @@ -74,7 +71,6 @@ sources: | order by type asc {add_query_items}' uri: None - parameters: list_detailed_virtual_machines: description: Retrieves list of VMs with network details metadata: diff --git a/msticpy/data/queries/splunk/splunk_queries.yaml b/msticpy/data/queries/splunk/splunk_queries.yaml index 0cdc60163..0d2a34231 100644 --- a/msticpy/data/queries/splunk/splunk_queries.yaml +++ b/msticpy/data/queries/splunk/splunk_queries.yaml @@ -47,7 +47,6 @@ sources: | fieldformat lastTime=strftime(lastTime,"%Y/%m/%d %H:%M:%S") | sort by totalCount desc ' - parameters: list_all_alerts: description: Retrieves all configured alerts metadata: @@ -57,7 +56,6 @@ sources: | rest/servicesNS/-/search/saved/searches | search alert.track=1 | fields title description search disabled triggered_alert_count actions action.script.filename alert.severity cron_schedule' - parameters: list_all_savedsearches: description: Retrieves all saved searches metadata: @@ -67,7 +65,6 @@ sources: | rest /servicesNS/-/-/saved/searches splunk_server=local | fields title, description, eai:acl.app, alert.severity, alert.track, alert_type, author, auto_summarize.cron_schedule, cron_schedule, id, is_scheduled, is_visible, qualifiedSearch, realtime_schedule, updated ' - parameters: list_all_audittrail: description: Retrieves all audit trail logs metadata: @@ -77,7 +74,6 @@ sources: search index={index} source=audittrail timeformat={timeformat} earliest={start} latest={end} | table timestamp, user, roles, action, info, search_type, is_realtime, savedsearch_name, search, total_run_time, event_count, result_count ' - parameters: get_events_parameterized: description: Generic parameterized query from index/source metadata: @@ -86,5 +82,4 @@ sources: query: ' search index={index} source={source} timeformat={timeformat} earliest={start} latest={end} {project_fields} - {add_query_items}' - parameters: \ No newline at end of file + {add_query_items}' \ No newline at end of file diff --git a/msticpy/data/queries/sumologic/sumologic_queries.yaml b/msticpy/data/queries/sumologic/sumologic_queries.yaml index d9b94132e..e2850dfcb 100644 --- a/msticpy/data/queries/sumologic/sumologic_queries.yaml +++ b/msticpy/data/queries/sumologic/sumologic_queries.yaml @@ -46,4 +46,3 @@ sources: | count _sourceCategory | sort -_count ' - parameters: diff --git a/tests/testdata/data_q_hierarchy.yaml b/tests/testdata/data_q_hierarchy.yaml index 26341d893..75df5cba9 100644 --- a/tests/testdata/data_q_hierarchy.yaml +++ b/tests/testdata/data_q_hierarchy.yaml @@ -55,7 +55,6 @@ sources: | project-away extendedProps {add_query_items}' uri: None - parameters: query2: description: Retrieves summary count of alerts by type metadata: @@ -71,7 +70,6 @@ sources: lastAlert=max(TimeGenerated) by AlertName | order by alertCount desc {add_query_items}' - parameters: query3: description: Retrieves a single alert by SystemAlertId metadata: diff --git a/tests/testdata/data_q_success.yaml b/tests/testdata/data_q_success.yaml index 1b7e4104a..717b6f486 100644 --- a/tests/testdata/data_q_success.yaml +++ b/tests/testdata/data_q_success.yaml @@ -53,7 +53,6 @@ sources: | project-away extendedProps {add_query_items}' uri: None - parameters: query2: description: Retrieves summary count of alerts by type metadata: @@ -68,7 +67,6 @@ sources: lastAlert=max(TimeGenerated) by AlertName | order by alertCount desc {add_query_items}' - parameters: query3: description: Retrieves a single alert by SystemAlertId metadata: diff --git a/tests/testdata/localdata/test_localdata_queries.yaml b/tests/testdata/localdata/test_localdata_queries.yaml index 08df3c7d3..7e372d670 100644 --- a/tests/testdata/localdata/test_localdata_queries.yaml +++ b/tests/testdata/localdata/test_localdata_queries.yaml @@ -21,21 +21,18 @@ sources: data_families: [Heartbeat] args: query: host_hb.csv - parameters: linux_events: description: List Linux Events metadata: data_families: [LinuxSyslog] args: query: linux_events.csv - parameters: linux_logons: description: List logons on host metadata: data_families: [LinuxSyslog] args: query: linux_logons.csv - parameters: list_host_logons: description: Retrieves the logon events on the host args: From a04df4d195b436364fbb347b031c7998806b6741 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 20 Jul 2023 22:06:19 +0200 Subject: [PATCH 12/29] Fixing typo --- msticpy/data/queries/mssentinel/kql_sent_net.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/msticpy/data/queries/mssentinel/kql_sent_net.yaml b/msticpy/data/queries/mssentinel/kql_sent_net.yaml index 6e95f1983..81d9424cf 100644 --- a/msticpy/data/queries/mssentinel/kql_sent_net.yaml +++ b/msticpy/data/queries/mssentinel/kql_sent_net.yaml @@ -277,7 +277,7 @@ sources: type: str default: "" ip_network_connections_csl: - descritpion: Returns network connections to and from an IP address (CommonSecurityLog) + description: Returns network connections to and from an IP address (CommonSecurityLog) metadata: pivot: short_name: ip_connections_csl @@ -302,7 +302,7 @@ sources: type: str default: "" all_network_connections_csl: - descritpion: Returns all network connections for a time range (CommonSecurityLog) + description: Returns all network connections for a time range (CommonSecurityLog) args: query: " {table} @@ -318,7 +318,7 @@ sources: type: str default: "CommonSecurityLog" ips_by_host_csl: - descritpion: Returns all IP addresses associated with a host (CommonSecurityLog) + description: Returns all IP addresses associated with a host (CommonSecurityLog) metadata: pivot: short_name: ips_csl @@ -344,7 +344,7 @@ sources: type: str default: "" hosts_by_ip_csl: - descritpion: Returns hosts associated with a IP addresses (CommonSecurityLog) + description: Returns hosts associated with a IP addresses (CommonSecurityLog) metadata: pivot: short_name: hosts_csl From 9e9c5532402ca34e4d1c130b4a263d524561ac7b Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 20 Jul 2023 22:06:31 +0200 Subject: [PATCH 13/29] Adding new metadata fields --- .schemas/queries.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.schemas/queries.json b/.schemas/queries.json index b3e1eabf5..b3577c7da 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -119,6 +119,21 @@ }, "cluster": { "type": "string" + }, + "clusters": { + "type": "array", + "items": { + "type": "string" + } + }, + "cluster_groups": { + "type": "array", + "items": { + "type": "string" + } + }, + "database": { + "type": "string" } }, "additionalProperties": false, From 09fc21cc7302574fb3a020c3369999c26f041747 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 20 Jul 2023 22:58:59 +0200 Subject: [PATCH 14/29] Remove empty parameters replace tabs with spaces --- tests/testdata/plugins/sql_test_queries.yaml | 27 +++----------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/tests/testdata/plugins/sql_test_queries.yaml b/tests/testdata/plugins/sql_test_queries.yaml index f50ff0741..88f345ea6 100644 --- a/tests/testdata/plugins/sql_test_queries.yaml +++ b/tests/testdata/plugins/sql_test_queries.yaml @@ -33,15 +33,15 @@ sources: let docAttachments = DeviceFileEvents | where Timestamp > minTimeRange - // Query for common document file extensions + // Query for common document file extensions and (FileName endswith ".docx" or FileName endswith ".docm" or FileName endswith ".doc") - // Query for files saved from email clients such as the Office Outlook app or the Windows Mail app + // Query for files saved from email clients such as the Office Outlook app or the Windows Mail app and InitiatingProcessFileName in~ ("outlook.exe", "hxoutlook.exe") | summarize AttachmentSaveTime=min(Timestamp) by AttachmentName=FileName, DeviceId; let browserDownloads = DeviceFileEvents | where Timestamp > minTimeRange - // Query for files created by common browsers + // Query for files created by common browsers and InitiatingProcessFileName in~ ("browser_broker.exe", "chrome.exe", "iexplore.exe", "firefox.exe") // Exclude JS files that are used for loading sites (but still query for JS files that are known to be downloaded) and not (FileName endswith ".js" and isempty(FileOriginUrl)) @@ -57,7 +57,6 @@ sources: by bin(Timestamp, 1tick), FileName, FileOriginUrl, ClickUrl, SHA1, DeviceName, DeviceId {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Doc%20attachment%20with%20link%20to%20download.txt" - parameters: dropbox_link: description: Looks for user content downloads from dropbox that originate from a link/redirect from a 3rd party site. metadata: @@ -72,7 +71,6 @@ sources: | project FileOriginReferrerUrl, FileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Dropbox%20downloads%20linked%20from%20other%20site.txt" - parameters: email_smartscreen: description: Look for links opened from outlook.exe, followed by a browser download and then a SmartScreen app warning metadata: @@ -108,7 +106,6 @@ sources: | distinct * {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Email%20link%20%2B%20download%20%2B%20SmartScreen%20warning.txt" - parameters: email_link: description: Look for links opened from mail apps – if a detection occurred right afterwards metadata: @@ -142,7 +139,6 @@ sources: | summarize FirstDetectedActivity=min(FirstDetectedActivity), AlertTitles=makeset(Title) by OpenedLink, InitiatingProcessFileName, Timestamp=bin(Timestamp, 1tick), DeviceName, DeviceId, WasOutlookSafeLink {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Open%20email%20link.txt" - parameters: av_sites: description: Pivot from downloads detected by Windows Defender Antivirus to other files downloaded from the same sites metadata: @@ -186,7 +182,6 @@ sources: | order by MachineCount desc {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Delivery/Pivot%20from%20detections%20to%20related%20downloads.txt" - parameters: tor: description: Looks for Tor client, or for a common Tor plugin called Meek. metadata: @@ -200,7 +195,6 @@ sources: | order by MachineCount desc {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Command%20and%20Control/Tor.txt" - parameters: network_scans: description: Looking for high volume queries against a given RemoteIP, per ComputerName, RemotePort and Process metadata: @@ -215,7 +209,6 @@ sources: | where RemotePortCount > remotePortCountThreshold {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Command%20and%20Control/Tor.txt" - parameters: user_enumeration: description: The query finds attempts to list users or groups using Net commands metadata: @@ -230,7 +223,6 @@ sources: | sort by AccountName, Target {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Discovery/Enumeration%20of%20users%20%26%20groups%20for%20lateral%20movement.txt" - parameters: smb_discovery: description: Query for processes that accessed more than 10 IP addresses over port 445 (SMB) - possibly scanning for network shares. metadata: @@ -244,7 +236,6 @@ sources: | where RemoteIPCount > 10 {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Discovery/SMB%20shares%20discovery.txt" - parameters: b64_pe: description: Finding base64 encoded PE files header seen in the command line parameters metadata: @@ -256,7 +247,6 @@ sources: | top 1000 by Timestamp {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/Base64encodePEFile.txt" - parameters: malware_recycle: description: Finding attackers hiding malware in the recycle bin. metadata: @@ -269,7 +259,6 @@ sources: | project Timestamp, DeviceName, ProcessCommandLine, InitiatingProcessFileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/Malware_In_recyclebin.txt" - parameters: powershell_downloads: description: Finds PowerShell execution events that could involve a download. metadata: @@ -290,7 +279,6 @@ sources: | top 100 by Timestamp {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Execution/PowerShell%20downloads.txt" - parameters: uncommon_powershell: description: Find which uncommon Powershell Cmdlets were executed on that machine in a certain time period. metadata: @@ -332,7 +320,6 @@ sources: | summarize FirstEvent=min(Timestamp), LastEvent=max(Timestamp) by DeviceName, ProcessCommandLine, FileName, InitiatingProcessFileName {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Electron-CVE-2018-1000006.txt" - parameters: cve_2018_4878: description: This query checks for specific processes and domain TLD used in the CVE-2018-4878 metadata: @@ -346,7 +333,6 @@ sources: | top 100 by Timestamp {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Flash-CVE-2018-4848.txt" - parameters: cve_2018_1111: description: Looks for CVE-2018-1111 exploitation metadata: @@ -360,7 +346,6 @@ sources: | project Timestamp, DeviceName , FileName, ProcessCommandLine, InitiatingProcessCommandLine {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Exploits/Linux-DynoRoot-CVE-2018-1111.txt" - parameters: brute_force: description: Look for public IP addresses that failed to logon to a computer multiple times, using multiple accounts, and eventually succeeded. metadata: @@ -382,7 +367,6 @@ sources: | where Failed > 10 and Successful > 0 and FailedAccountsCount > 2 and SuccessfulAccountsCount == 1 {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Lateral%20Movement/Account%20brute%20force.txt" - parameters: service_account_powershell: description: Service Accounts Performing Remote PowerShell metadata: @@ -428,7 +412,6 @@ sources: | where min_Timestamp > ago(1d) {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Lateral%20Movement/ServiceAccountsPerformingRemotePS.txt" - parameters: accessibility_persistence: description: This query looks for persistence or privilege escalation done using Windows Accessibility features. metadata: @@ -474,7 +457,6 @@ sources: | union kind=outer executedProcessIsPowershellOrCmd {add_query_items}' uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Persistence/Accessibility%20Features.txt" - parameters: smartscreen_ignored: description: Query for SmartScreen URL blocks, where the user has decided to run the malware nontheless. metadata: @@ -505,5 +487,4 @@ sources: | where DownloadTime - Timestamp between (0min .. 2min) | project-away DeviceName1 {add_query_items}' - uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Protection%20events/SmartScreen%20URL%20block%20ignored%20by%20user.txt" - parameters: \ No newline at end of file + uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Protection%20events/SmartScreen%20URL%20block%20ignored%20by%20user.txt" \ No newline at end of file From 900416f168af92599d94678d15c98de8ac54da6f Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 20 Jul 2023 22:59:24 +0200 Subject: [PATCH 15/29] Add SQL data environment --- .schemas/queries.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.schemas/queries.json b/.schemas/queries.json index b3577c7da..617a3f905 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -88,7 +88,9 @@ "ResourceGraph", "SecurityGraph", "Splunk", - "Sumologic" + "Sumologic", + "SQLTestProvider", + "SQLProdProvider" ] } }, From ac6036751c35523eaecc2082b8ce3023ccf51ef4 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 20 Jul 2023 22:59:41 +0200 Subject: [PATCH 16/29] Ignore folder containing Sentinel queries --- tests/common/test_pkg_config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index a7de5b3bc..4cd5a01bd 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -63,7 +63,11 @@ def test_load_default(): check.is_true(path.is_dir()) for query_file in Path(path).resolve().rglob("*.yaml"): validate_queries_file_structure( - query_file, expected=not ("fail" in query_file.name) + query_file, + expected=not ( + "fail" in query_file.name + or "sentinel_query_import_data" in query_file.absolute + ), ) From 051ca617ccb979860db83ac752320be0107b243c Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 1 Aug 2023 21:56:20 +0200 Subject: [PATCH 17/29] Fix call to absolute --- tests/common/test_pkg_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index 4cd5a01bd..ac6b65e35 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -66,7 +66,7 @@ def test_load_default(): query_file, expected=not ( "fail" in query_file.name - or "sentinel_query_import_data" in query_file.absolute + or "sentinel_query_import_data" in query_file.absolute() ), ) From bf87c1f6b8796ddca4316879cbc1c762fb0e88f3 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 1 Aug 2023 21:56:41 +0200 Subject: [PATCH 18/29] Update JSON validation schema to have an open list of providers --- .schemas/queries.json | 50 +++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/.schemas/queries.json b/.schemas/queries.json index 617a3f905..b08dbe9d7 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -68,29 +68,33 @@ "data_environments": { "type": "array", "items": { - "type": "string", - "enum": [ - "AzureSecurityCenter", - "AzureSentinel", - "Cybereason", - "Elastic", - "Kusto", - "LocalData", - "LogAnalytics", - "M365D", - "MDATP", - "MDE", - "Mordor", - "MSGraph", - "MSSentinel", - "OSQueryLogs", - "OTRF", - "ResourceGraph", - "SecurityGraph", - "Splunk", - "Sumologic", - "SQLTestProvider", - "SQLProdProvider" + "oneOf": [ + { + "enum": [ + "AzureSecurityCenter", + "AzureSentinel", + "Cybereason", + "Elastic", + "Kusto", + "LocalData", + "LogAnalytics", + "M365D", + "MDATP", + "MDE", + "Mordor", + "MSGraph", + "MSSentinel", + "OSQueryLogs", + "OTRF", + "ResourceGraph", + "SecurityGraph", + "Splunk", + "Sumologic" + ] + }, + { + "type": "string" + } ] } }, From 77acc04aa976d362354aa43f194686b9fa8e3b01 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 1 Aug 2023 22:11:03 +0200 Subject: [PATCH 19/29] Replacing tabs with spaces --- msticpy/data/queries/m365d/kql_m365_hunting.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msticpy/data/queries/m365d/kql_m365_hunting.yaml b/msticpy/data/queries/m365d/kql_m365_hunting.yaml index f7d671f39..69dfb3f36 100644 --- a/msticpy/data/queries/m365d/kql_m365_hunting.yaml +++ b/msticpy/data/queries/m365d/kql_m365_hunting.yaml @@ -37,15 +37,15 @@ sources: let docAttachments = DeviceFileEvents | where {time_column} > minTimeRange - // Query for common document file extensions + // Query for common document file extensions and (FileName endswith ".docx" or FileName endswith ".docm" or FileName endswith ".doc") - // Query for files saved from email clients such as the Office Outlook app or the Windows Mail app + // Query for files saved from email clients such as the Office Outlook app or the Windows Mail app and InitiatingProcessFileName in~ ("outlook.exe", "hxoutlook.exe") | summarize AttachmentSaveTime=min({time_column}) by AttachmentName=FileName, DeviceId; let browserDownloads = DeviceFileEvents | where {time_column} > minTimeRange - // Query for files created by common browsers + // Query for files created by common browsers and InitiatingProcessFileName in~ ("browser_broker.exe", "chrome.exe", "iexplore.exe", "firefox.exe") // Exclude JS files that are used for loading sites (but still query for JS files that are known to be downloaded) and not (FileName endswith ".js" and isempty(FileOriginUrl)) From 6732d7b82e9d3c84c5fb1c4b92345fa81952354b Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 1 Aug 2023 22:11:16 +0200 Subject: [PATCH 20/29] s/oneOf/anyOf/ --- .schemas/queries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.schemas/queries.json b/.schemas/queries.json index b08dbe9d7..9499dd981 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -68,7 +68,7 @@ "data_environments": { "type": "array", "items": { - "oneOf": [ + "anyOf": [ { "enum": [ "AzureSecurityCenter", From d0d8e8d7743482eaf6351ad28a509728f1db5b89 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 1 Aug 2023 22:32:21 +0200 Subject: [PATCH 21/29] Rework condition to ignore sentinel_query_import_data --- tests/common/test_pkg_config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index ac6b65e35..c4198cbe2 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -66,7 +66,13 @@ def test_load_default(): query_file, expected=not ( "fail" in query_file.name - or "sentinel_query_import_data" in query_file.absolute() + or query_file.absolute().is_relative_to( + Path(__file__) + .resolve() + .parent.parent.joinpath( + "testdata", "sentinel_query_import_data" + ) + ) ), ) From e32f31f7ec447e4065204e027387d2ba15582d64 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 1 Aug 2023 22:43:15 +0200 Subject: [PATCH 22/29] Simplify comparison for python 3.8 --- tests/common/test_pkg_config.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index c4198cbe2..77e148723 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -66,13 +66,8 @@ def test_load_default(): query_file, expected=not ( "fail" in query_file.name - or query_file.absolute().is_relative_to( - Path(__file__) - .resolve() - .parent.parent.joinpath( - "testdata", "sentinel_query_import_data" - ) - ) + or "sentinel_query_import_data" + in [parent.name for parent in query_file.absolute().parents] ), ) From b0d26960469d549d558e6a5eb322a7e918549dc2 Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Thu, 3 Aug 2023 08:45:01 +0000 Subject: [PATCH 23/29] Increasing the size of description to 1024 characters --- .schemas/queries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.schemas/queries.json b/.schemas/queries.json index 9499dd981..0693a9499 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -54,7 +54,7 @@ "description": { "type": "string", "minLength": 5, - "maxLength": 256 + "maxLength": 1024 }, "metadata": { "type": "object", From f75587b5afc08f2fbc808d234fa78aa1e4685620 Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Fri, 18 Aug 2023 16:03:04 +0000 Subject: [PATCH 24/29] Adding metadata, sources and defaults as mandatory --- .schemas/queries.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.schemas/queries.json b/.schemas/queries.json index 0693a9499..4f5575308 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -50,6 +50,11 @@ } } }, + "required": [ + "metadata", + "defaults", + "sources" + ], "$defs": { "description": { "type": "string", From 8903a70273a8fb1193fa56cd2d3b7ba972c1cdc4 Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Fri, 18 Aug 2023 16:19:31 +0000 Subject: [PATCH 25/29] Ignore msticpyconfig files --- tests/common/test_pkg_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index 77e148723..cb312858c 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -66,6 +66,7 @@ def test_load_default(): query_file, expected=not ( "fail" in query_file.name + or "msticpyconfig" in query_file.name or "sentinel_query_import_data" in [parent.name for parent in query_file.absolute().parents] ), From fd7dd451671492c6dde4f261f3a5c129685c6ebe Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Tue, 5 Sep 2023 08:50:09 +0000 Subject: [PATCH 26/29] Remove defaults as a required key --- .schemas/queries.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.schemas/queries.json b/.schemas/queries.json index 4f5575308..f1d0e8d91 100644 --- a/.schemas/queries.json +++ b/.schemas/queries.json @@ -52,7 +52,6 @@ }, "required": [ "metadata", - "defaults", "sources" ], "$defs": { From 0c67cc7b4a08925608c55c66a45f499f3b2a081e Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Tue, 5 Sep 2023 08:50:56 +0000 Subject: [PATCH 27/29] Created dedicated tests for query validation --- tests/data/queries/__init__.py | 0 tests/data/queries/test_query_files.py | 45 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/data/queries/__init__.py create mode 100644 tests/data/queries/test_query_files.py diff --git a/tests/data/queries/__init__.py b/tests/data/queries/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/queries/test_query_files.py b/tests/data/queries/test_query_files.py new file mode 100644 index 000000000..749cfcfbd --- /dev/null +++ b/tests/data/queries/test_query_files.py @@ -0,0 +1,45 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""Query files test class.""" +from pathlib import Path + +import pytest_check as check +import yaml +from jsonschema import Draft7Validator + +from ...unit_test_lib import get_queries_schema, get_test_data_path + +_QUERIES_SCHEMA = get_queries_schema() + + +def validate_queries_file_structure(query_file: Path, expected: bool = True): + """Test if query files have a valid structure.""" + with query_file.open(mode="r", encoding="utf-8") as queries: + queries_yaml = yaml.safe_load(queries) + if expected: + check.is_true( + Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), + msg=f"File {query_file} is not a valid query file", + ) + else: + check.is_false( + Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), + msg=f"File {query_file} was expected to be invalid but is a valid query file", + ) + + +def test_valid_queries(): + """Test load default settings.""" + valid_queries_path = get_test_data_path() / "queries" / "valid" + for valid_query in valid_queries_path.rglob("*yaml"): + validate_queries_file_structure(valid_query) + + +def test_invalid_queries(): + """Test load default settings.""" + valid_queries_path = get_test_data_path() / "queries" / "invalid" + for valid_query in valid_queries_path.rglob("*yaml"): + validate_queries_file_structure(valid_query, expected=False) From 28305807618a90841e9e1cacf52e11547d0300af Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Tue, 5 Sep 2023 08:51:14 +0000 Subject: [PATCH 28/29] Add multiple sample of valid and invalid queries --- ...ample_minimal_empty_parameter_default.yaml | 19 +++++ .../sample_minimal_incorrect_version.yaml | 13 +++ ...sample_minimal_invalid_parameter_type.yaml | 18 +++++ ...le_minimal_long_parameter_description.yaml | 21 +++++ .../sample_minimal_missing_metadata.yaml | 5 ++ .../sample_minimal_missing_sources.yaml | 8 ++ ...e_minimal_short_parameter_description.yaml | 18 +++++ .../queries/valid/sample_complete.yaml | 81 +++++++++++++++++++ .../queries/valid/sample_minimal.yaml | 13 +++ 9 files changed, 196 insertions(+) create mode 100644 tests/testdata/queries/invalid/sample_minimal_empty_parameter_default.yaml create mode 100644 tests/testdata/queries/invalid/sample_minimal_incorrect_version.yaml create mode 100644 tests/testdata/queries/invalid/sample_minimal_invalid_parameter_type.yaml create mode 100644 tests/testdata/queries/invalid/sample_minimal_long_parameter_description.yaml create mode 100644 tests/testdata/queries/invalid/sample_minimal_missing_metadata.yaml create mode 100644 tests/testdata/queries/invalid/sample_minimal_missing_sources.yaml create mode 100644 tests/testdata/queries/invalid/sample_minimal_short_parameter_description.yaml create mode 100644 tests/testdata/queries/valid/sample_complete.yaml create mode 100644 tests/testdata/queries/valid/sample_minimal.yaml diff --git a/tests/testdata/queries/invalid/sample_minimal_empty_parameter_default.yaml b/tests/testdata/queries/invalid/sample_minimal_empty_parameter_default.yaml new file mode 100644 index 000000000..395e7edfe --- /dev/null +++ b/tests/testdata/queries/invalid/sample_minimal_empty_parameter_default.yaml @@ -0,0 +1,19 @@ +metadata: + version: 1 + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 +defaults: + parameters: + parameter1: + description: First Parameter + type: str + default: +sources: + query1: + description: Description for the first query + args: + query: MyQuery diff --git a/tests/testdata/queries/invalid/sample_minimal_incorrect_version.yaml b/tests/testdata/queries/invalid/sample_minimal_incorrect_version.yaml new file mode 100644 index 000000000..0191e921b --- /dev/null +++ b/tests/testdata/queries/invalid/sample_minimal_incorrect_version.yaml @@ -0,0 +1,13 @@ +metadata: + version: "1" + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 +sources: + query1: + description: Description for the first query + args: + query: MyQuery diff --git a/tests/testdata/queries/invalid/sample_minimal_invalid_parameter_type.yaml b/tests/testdata/queries/invalid/sample_minimal_invalid_parameter_type.yaml new file mode 100644 index 000000000..3367f5c12 --- /dev/null +++ b/tests/testdata/queries/invalid/sample_minimal_invalid_parameter_type.yaml @@ -0,0 +1,18 @@ +metadata: + version: 1 + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 +defaults: + parameters: + parameter1: + description: First Parameter + type: string +sources: + query1: + description: Description for the first query + args: + query: MyQuery diff --git a/tests/testdata/queries/invalid/sample_minimal_long_parameter_description.yaml b/tests/testdata/queries/invalid/sample_minimal_long_parameter_description.yaml new file mode 100644 index 000000000..0a208f92e --- /dev/null +++ b/tests/testdata/queries/invalid/sample_minimal_long_parameter_description.yaml @@ -0,0 +1,21 @@ +metadata: + version: 1 + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 +defaults: + parameters: + parameter1: + description: |- + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce in felis tincidunt, posuere libero vehicula, accumsan ipsum. Nullam molestie dolor eu tellus sollicitudin consequat. Duis lobortis facilisis sapien, sit amet sagittis urna tempus eget. Donec luctus, erat in egestas mattis, tellus nunc elementum felis, ac suscipit lectus sapien ut magna. Donec id purus et urna cursus cursus nec eget neque. Donec id feugiat arcu. Ut aliquam et ipsum non tincidunt. Sed vehicula imperdiet mi, a ultrices ante vestibulum vitae. Sed vestibulum tortor tortor, a consequat dui ornare sed. In dui nibh, congue vitae nibh ut, facilisis suscipit ipsum. Integer iaculis neque in lobortis volutpat. Suspendisse tristique molestie nisi, nec molestie massa pulvinar sit amet. + + Proin sapien nulla, aliquet rhoncus libero eget, rhoncus eleifend arcu. Vestibulum vitae neque sollicitudin orci ultricies varius. Sed id eros elementum, molestie sapien et, mollis purus. Ut ac ultrices sapien, ac efficitur ipsum. Nunc nec aliquet mauris, nec fermentum arcu. Nulla facilisi. Morbi non commodo nulla. Fusce quis leo vitae leo commodo pulvinar vitae eget turpis. Vestibulum ultricies mauris quis urna facilisis, nec vestibulum enim cursus. Suspendisse ac convallis ligula. Nam non velit iaculis, viverra orci a, ornare leo. Sed ac porttitor risus. + type: str +sources: + query1: + description: Description for the first query + args: + query: MyQuery diff --git a/tests/testdata/queries/invalid/sample_minimal_missing_metadata.yaml b/tests/testdata/queries/invalid/sample_minimal_missing_metadata.yaml new file mode 100644 index 000000000..0884f4149 --- /dev/null +++ b/tests/testdata/queries/invalid/sample_minimal_missing_metadata.yaml @@ -0,0 +1,5 @@ +sources: + query1: + description: Description for the first query + args: + query: MyQuery diff --git a/tests/testdata/queries/invalid/sample_minimal_missing_sources.yaml b/tests/testdata/queries/invalid/sample_minimal_missing_sources.yaml new file mode 100644 index 000000000..701c62ade --- /dev/null +++ b/tests/testdata/queries/invalid/sample_minimal_missing_sources.yaml @@ -0,0 +1,8 @@ +metadata: + version: 1 + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 diff --git a/tests/testdata/queries/invalid/sample_minimal_short_parameter_description.yaml b/tests/testdata/queries/invalid/sample_minimal_short_parameter_description.yaml new file mode 100644 index 000000000..1a7c2b2b6 --- /dev/null +++ b/tests/testdata/queries/invalid/sample_minimal_short_parameter_description.yaml @@ -0,0 +1,18 @@ +metadata: + version: 1 + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 +defaults: + parameters: + parameter1: + description: A + type: str +sources: + query1: + description: Description for the first query + args: + query: MyQuery diff --git a/tests/testdata/queries/valid/sample_complete.yaml b/tests/testdata/queries/valid/sample_complete.yaml new file mode 100644 index 000000000..53238d342 --- /dev/null +++ b/tests/testdata/queries/valid/sample_complete.yaml @@ -0,0 +1,81 @@ +metadata: + version: 1 + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 + tags: + - alert + - securityalert + aliases: MyAlias + cluster: MyCluster + clusters: + - MyCluster1 + - MyCluster2 + cluster_groups: + - MyClusterGroup1 + - MyClusterGroup2 + database: MyDatabase +defaults: + metadata: + data_source: security_alert + data_families: + - DataFamily1 + - DataFamily2 + pivot: + myPivot: + - pivot1 + top: 42 + parameters: + parameter1: + description: First Parameter + type: str + default: Default Value for Parameter 1 + aliases: + - param1 + - p1 + parameter2: + description: Second Parameter + type: datetime + default: Default Value for Parameter 2 + aliases: param2 + parameter3: + description: Third Parameter + type: int + parameter4: + description: Fourth Parameter + type: float +sources: + query1: + description: Description for the first query + metadata: + version: 1 + description: Fancy description for a sample query + data_environments: MyEnvironment + data_families: + - SecurityAlert + tags: + - alert + - securityalert + aliases: MyAlias + cluster: MyCluster + clusters: + - MyCluster1 + - MyCluster2 + cluster_groups: + - MyClusterGroup1 + - MyClusterGroup2 + database: MyDatabase + args: + query: MyQuery + uri: https://microsoft.com/ + query_macros: + macro1: + description: Macro Description + value: MyMacroValue + parameters: + parameter5: + description: Fifth Parameter + type: list \ No newline at end of file diff --git a/tests/testdata/queries/valid/sample_minimal.yaml b/tests/testdata/queries/valid/sample_minimal.yaml new file mode 100644 index 000000000..b277e9c54 --- /dev/null +++ b/tests/testdata/queries/valid/sample_minimal.yaml @@ -0,0 +1,13 @@ +metadata: + version: 1 + description: Fancy description for a sample query + data_environments: + - MyEnvironment + data_families: + - Data Family 1 + - Data Family 2 +sources: + query1: + description: Description for the first query + args: + query: MyQuery From 1c6bc58a416b9dc4c9917b4cb82fdb115078e5f8 Mon Sep 17 00:00:00 2001 From: Florian Bracq Date: Tue, 5 Sep 2023 08:51:44 +0000 Subject: [PATCH 29/29] Update test_pkg_config based on commit's comments --- tests/common/test_pkg_config.py | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/tests/common/test_pkg_config.py b/tests/common/test_pkg_config.py index cb312858c..48cbe4f94 100644 --- a/tests/common/test_pkg_config.py +++ b/tests/common/test_pkg_config.py @@ -9,7 +9,6 @@ from pathlib import Path import httpx -from jsonschema import Draft7Validator import pytest import pytest_check as check import yaml @@ -17,30 +16,14 @@ from msticpy.common import pkg_config from msticpy.context.geoip import GeoLiteLookup, IPStackLookup -from ..unit_test_lib import custom_mp_config, get_test_data_path, get_queries_schema +from ..unit_test_lib import custom_mp_config, get_test_data_path +from ..data.queries.test_query_files import validate_queries_file_structure _TEST_DATA = get_test_data_path() -_QUERIES_SCHEMA = get_queries_schema() # pylint: disable=protected-access -def validate_queries_file_structure(query_file: Path, expected: bool = True): - """Test if query files have a valid structure.""" - with query_file.open(mode="r", encoding="utf-8") as queries: - queries_yaml = yaml.safe_load(queries) - if expected: - check.is_true( - Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), - msg=f"File {query_file} is not a valid query file", - ) - else: - check.is_false( - Draft7Validator(_QUERIES_SCHEMA).is_valid(queries_yaml), - msg=f"File {query_file} was expected to be invalid but is a valid query file", - ) - - def test_load_default(): """Test load default settings.""" settings = pkg_config._settings @@ -62,15 +45,8 @@ def test_load_default(): path = Path(__file__).resolve().parent.parent.joinpath(path) check.is_true(path.is_dir()) for query_file in Path(path).resolve().rglob("*.yaml"): - validate_queries_file_structure( - query_file, - expected=not ( - "fail" in query_file.name - or "msticpyconfig" in query_file.name - or "sentinel_query_import_data" - in [parent.name for parent in query_file.absolute().parents] - ), - ) + if "tests" not in [parent.name for parent in query_file.absolute().parents]: + validate_queries_file_structure(query_file) def test_custom_config():