diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa8b414..3f960451 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes staged on develop +## Improvements +- Clarified documentation on creation of virtual environments for Windows compatibility. +- Added section to install docs about taxii2client compatibility. See issue [#19](https://github.com/mitre-attack/attack-scripts/issues/19). +- Standardized import style to better follow the standard set in [MITRE/CTI#103](https://github.com/mitre/cti/pull/103). + # v1.5.1 - 27 July 2020 ## New Scripts - Added [layer to excel](https://github.com/mitre-attack/attack-scripts/tree/master/layers#to_excelpy) converter. See issue [#25](https://github.com/mitre-attack/attack-scripts/issues/25). diff --git a/README.md b/README.md index f11833b2..803ac1e6 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,20 @@ This repository contains various tools and utilities for working with ATT&CK con - [python3](https://www.python.org/) ## Installation -1. Create a new virtual environment: `python3 -m venv env` -2. Activate the environment: `source env/bin/activate` +1. Create virtual environment: + - macOS and Linux: `python3 -m venv env` + - Windows: `py -m venv env` +2. Activate the virtual environment: + - macOS and Linux: `source env/bin/activate` + - Windows: `env/Scripts/activate.bat` 3. Install requirements into the virtual environment: `pip3 install -r requirements.txt` +### taxii2client compatibility +A recent update to the `taxii2client` python library changed the API for TAXII 2.0 requests. This repository has been updated to work with the latest version of taxii2client, however if your installed version is < 2.0.0 you may need to patch the scripts for them to run properly. Please see our [USAGE document on MITRE/CTI](https://github.com/mitre/cti/blob/master/USAGE.md#taxii2client) for more information. + ## Related MITRE Work #### CTI -[Cyber Threat Intelligence repository](https://github.com/mitre/cti) of the ATT&CK catalog expressed in STIX 2.0 JSON. +[Cyber Threat Intelligence repository](https://github.com/mitre/cti) of the ATT&CK catalog expressed in STIX 2.0 JSON. This repository also contains [our USAGE document](https://github.com/mitre/cti/blob/master/USAGE.md) which includes additional examples of accessing and parsing our dataset in Python. #### ATT&CK ATT&CK® is a curated knowledge base and model for cyber adversary behavior, reflecting the various phases of an adversary’s lifecycle and the platforms they are known to target. ATT&CK is useful for understanding security risk against known adversary behavior, for planning security improvements, and verifying defenses work as expected. diff --git a/requirements.txt b/requirements.txt index a84eceba..8386057b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,31 @@ antlr4-python3-runtime==4.7.2 +cairocffi==1.1.0 +CairoSVG==2.4.2 certifi==2018.11.29 +cffi==1.14.1 chardet==3.0.4 +colour==0.1.5 +cssselect2==0.3.0 +defusedxml==0.6.0 +drawSvg==1.6.0 +et-xmlfile==1.0.1 idna==2.8 +imageio==2.9.0 +jdcal==1.4.1 +numpy==1.19.1 +openpyxl==3.0.3 +Pillow==7.2.0 +pycparser==2.20 python-dateutil==2.8.0 pytz==2018.9 requests==2.21.0 simplejson==3.16.0 -six==1.12.0 -stix2==1.1.2 +six==1.15.0 +stix2==2.0.2 stix2-patterns==1.1.0 tabulate==0.8.3 -taxii2-client==0.5.0 +taxii2-client==2.2.1 +tinycss2==1.0.2 tqdm==4.31.1 urllib3==1.24.2 -colour==0.1.5 -openpyxl==3.0.3 \ No newline at end of file +webencodings==0.5.1 diff --git a/scripts/diff_stix.py b/scripts/diff_stix.py index 96c20670..d1930267 100644 --- a/scripts/diff_stix.py +++ b/scripts/diff_stix.py @@ -1,6 +1,6 @@ import argparse from stix2 import MemoryStore, Filter, TAXIICollectionSource -from taxii2client import Collection +from taxii2client.v20 import Collection import os import json from tqdm import tqdm diff --git a/scripts/layers/samples/apt3_apt29_software.py b/scripts/layers/samples/apt3_apt29_software.py index a4a41c21..3901b35e 100644 --- a/scripts/layers/samples/apt3_apt29_software.py +++ b/scripts/layers/samples/apt3_apt29_software.py @@ -1,7 +1,7 @@ import argparse import requests import json -import stix2 +from stix2 import MemoryStore def generate(show_nodetect=False): """ @@ -9,7 +9,7 @@ def generate(show_nodetect=False): param show_nodetect, if true, causes techniques that have no data-sources to be highlighted as well """ stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json() - ms = stix2.MemoryStore(stix_data=stix["objects"]) + ms = MemoryStore(stix_data=stix["objects"]) apt3 = ms.get("intrusion-set--0bbdf25b-30ff-4894-a1cd-49260d0dd2d9") apt29 = ms.get("intrusion-set--899ce53f-13a0-479b-a0e4-67d46e241542") diff --git a/scripts/layers/samples/bear_APT.py b/scripts/layers/samples/bear_APT.py index be2abd1c..e62d3bbb 100644 --- a/scripts/layers/samples/bear_APT.py +++ b/scripts/layers/samples/bear_APT.py @@ -1,16 +1,16 @@ import argparse import requests import json -import stix2 +from stix2 import MemoryStore, Filter import re def generate(): """parse the STIX on MITRE/CTI and return a layer dict showing all techniques used by an APT group with phrase 'bear' in the group aliases.""" # import the STIX data from MITRE/CTI stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json() - ms = stix2.MemoryStore(stix_data=stix["objects"]) + ms = MemoryStore(stix_data=stix["objects"]) - groups = ms.query([ stix2.Filter("type", "=", "intrusion-set") ]) + groups = ms.query([ Filter("type", "=", "intrusion-set") ]) # find bear groups bear_groups = [] #list of groups with bear in name diff --git a/scripts/layers/samples/heatmap.py b/scripts/layers/samples/heatmap.py index 99e95eb6..75bd64d2 100644 --- a/scripts/layers/samples/heatmap.py +++ b/scripts/layers/samples/heatmap.py @@ -1,17 +1,17 @@ import argparse import requests import json -import stix2 +from stix2 import MemoryStore, Filter import random def generate(): """parse the STIX on MITRE/CTI and return a layer dict with techniques with randomized scores""" # import the STIX data from MITRE/CTI stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json() - ms = stix2.MemoryStore(stix_data=stix["objects"]) + ms = MemoryStore(stix_data=stix["objects"]) # get all techniques in STIX techniques = ms.query([ - stix2.Filter("type", "=", "attack-pattern") + Filter("type", "=", "attack-pattern") ]) # parse techniques into layer format techniques_list = [] diff --git a/scripts/layers/samples/software_execution.py b/scripts/layers/samples/software_execution.py index e13fb493..df49839f 100644 --- a/scripts/layers/samples/software_execution.py +++ b/scripts/layers/samples/software_execution.py @@ -1,7 +1,7 @@ import argparse import requests import json -import stix2 +from stix2 import Filter, MemoryStore from itertools import chain def generate(softwaretype="software"): @@ -10,13 +10,13 @@ def generate(softwaretype="software"): """ # import the STIX data from MITRE/CTI stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json() - ms = stix2.MemoryStore(stix_data=stix["objects"]) + ms = MemoryStore(stix_data=stix["objects"]) # software includes malware and tool types so perform two queries and merge the results software_filters = [] if softwaretype == "malware" or softwaretype == "software": - software_filters.append( [ stix2.Filter('type', '=', 'malware') ] ) + software_filters.append( [ Filter('type', '=', 'malware') ] ) if softwaretype == "tool" or softwaretype == "software": - software_filters.append( [ stix2.Filter('type', '=', 'tool') ] ) + software_filters.append( [ Filter('type', '=', 'tool') ] ) software = list(chain.from_iterable( ms.query(f) for f in software_filters diff --git a/scripts/technique_mappings_to_csv.py b/scripts/technique_mappings_to_csv.py index fb90420b..29161437 100644 --- a/scripts/technique_mappings_to_csv.py +++ b/scripts/technique_mappings_to_csv.py @@ -2,8 +2,9 @@ import csv import io -import stix2 -import taxii2client +from stix2 import TAXIICollectionSource, MemorySource, Filter +from taxii2client.v20 import Collection + import tqdm @@ -15,18 +16,18 @@ def build_taxii_source(collection_name): "mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b" } collection_url = "https://cti-taxii.mitre.org/stix/collections/" + collection_map[collection_name] + "/" - collection = taxii2client.Collection(collection_url) - taxii_ds = stix2.TAXIICollectionSource(collection) + collection = Collection(collection_url) + taxii_ds = TAXIICollectionSource(collection) # Create an in-memory source (to prevent multiple web requests) - return stix2.MemorySource(stix_data=taxii_ds.query()) + return MemorySource(stix_data=taxii_ds.query()) def get_all_techniques(src, source_name): """Filters data source by attack-pattern which extracts all ATT&CK Techniques""" filters = [ - stix2.Filter("type", "=", "attack-pattern"), - stix2.Filter("external_references.source_name", "=", source_name), + Filter("type", "=", "attack-pattern"), + Filter("external_references.source_name", "=", source_name), ] results = src.query(filters) return remove_deprecated(results) @@ -35,13 +36,13 @@ def get_all_techniques(src, source_name): def filter_for_term_relationships(src, relationship_type, object_id, target=True): """Filters data source by type, relationship_type and source or target""" filters = [ - stix2.Filter("type", "=", "relationship"), - stix2.Filter("relationship_type", "=", relationship_type), + Filter("type", "=", "relationship"), + Filter("relationship_type", "=", relationship_type), ] if target: - filters.append(stix2.Filter("target_ref", "=", object_id)) + filters.append(Filter("target_ref", "=", object_id)) else: - filters.append(stix2.Filter("source_ref", "=", object_id)) + filters.append(Filter("source_ref", "=", object_id)) results = src.query(filters) return remove_deprecated(results) @@ -50,9 +51,9 @@ def filter_for_term_relationships(src, relationship_type, object_id, target=True def filter_by_type_and_id(src, object_type, object_id, source_name): """Filters data source by id and type""" filters = [ - stix2.Filter("type", "=", object_type), - stix2.Filter("id", "=", object_id), - stix2.Filter("external_references.source_name", "=", source_name), + Filter("type", "=", object_type), + Filter("id", "=", object_id), + Filter("external_references.source_name", "=", source_name), ] results = src.query(filters) return remove_deprecated(results) diff --git a/scripts/techniques_data_sources_vis.py b/scripts/techniques_data_sources_vis.py index e10d93ad..371265d9 100644 --- a/scripts/techniques_data_sources_vis.py +++ b/scripts/techniques_data_sources_vis.py @@ -1,6 +1,6 @@ import json, os, shutil, sys from stix2 import TAXIICollectionSource, Filter -import taxii2client as t2c +from taxii2client.v20 import Collection from pprint import pprint import argparse from tqdm import tqdm @@ -94,19 +94,11 @@ def establish_connection(collection: str): connection to the taxii collection """ - try: - with open('proxy.json', "r") as json_data: - proxies = json.load(json_data) - link = t2c._HTTPConnection() - link.session.proxies.update(proxies) - collection = t2c.Collection(collection, conn=link) - tc_src = TAXIICollectionSource(collection) - except: - if verbose: print("unable to find proxy file, falling back to standard callout... ", end="", flush=True) - # Establish TAXII2 Collection instance for Enterprise ATT&CK collection - collection = t2c.Collection(collection) - # Supply the collection to TAXIICollection - tc_src = TAXIICollectionSource(collection) + + # Establish TAXII2 Collection instance for Enterprise ATT&CK collection + collection = Collection(collection) + # Supply the collection to TAXIICollection + tc_src = TAXIICollectionSource(collection) return tc_src diff --git a/scripts/techniques_from_data_source.py b/scripts/techniques_from_data_source.py index bc4b51e4..b3982dbb 100644 --- a/scripts/techniques_from_data_source.py +++ b/scripts/techniques_from_data_source.py @@ -1,5 +1,5 @@ from stix2 import TAXIICollectionSource, Filter -from taxii2client import Collection +from taxii2client.v20 import Collection import argparse # Establish TAXII2 Collection instance for Enterprise ATT&CK collection