improve documentation, update imports

CHANGELOG.md

@@ -1,3 +1,9 @@
+# Changes staged on develop
+## Improvements
+- Clarified documentation on creation of virtual environments for Windows compatibility.
+- Added section to install docs about taxii2client compatibility. See issue [#19](https://github.com/mitre-attack/attack-scripts/issues/19).
+- Standardized import style to better follow the standard set in [MITRE/CTI#103](https://github.com/mitre/cti/pull/103).
+
 # v1.5.1 - 27 July 2020
 ## New Scripts
 - Added [layer to excel](https://github.com/mitre-attack/attack-scripts/tree/master/layers#to_excelpy) converter. See issue [#25](https://github.com/mitre-attack/attack-scripts/issues/25).

README.md

@@ -9,13 +9,20 @@ This repository contains various tools and utilities for working with ATT&CK con
 - [python3](https://www.python.org/)
 
 ## Installation
-1. Create a new virtual environment: `python3 -m venv env`
-2. Activate the environment: `source env/bin/activate`
+1. Create virtual environment:
+    - macOS and Linux: `python3 -m venv env`
+    - Windows: `py -m venv env`
+2. Activate the virtual environment:
+    - macOS and Linux: `source env/bin/activate`
+    - Windows: `env/Scripts/activate.bat`
 3. Install requirements into the virtual environment: `pip3 install -r requirements.txt`
 
+### taxii2client compatibility
+A recent update to the `taxii2client` python library changed the API for TAXII 2.0 requests. This repository has been updated to work with the latest version of taxii2client, however if your installed version is < 2.0.0 you may need to patch the scripts for them to run properly. Please see our [USAGE document on MITRE/CTI](https://github.com/mitre/cti/blob/master/USAGE.md#taxii2client) for more information.
+
 ## Related MITRE Work
 #### CTI
-[Cyber Threat Intelligence repository](https://github.com/mitre/cti) of the ATT&CK catalog expressed in STIX 2.0 JSON.
+[Cyber Threat Intelligence repository](https://github.com/mitre/cti) of the ATT&CK catalog expressed in STIX 2.0 JSON. This repository also contains [our USAGE document](https://github.com/mitre/cti/blob/master/USAGE.md) which includes additional examples of accessing and parsing our dataset in Python.
 
 #### ATT&CK
 ATT&CK® is a curated knowledge base and model for cyber adversary behavior, reflecting the various phases of an adversary’s lifecycle and the platforms they are known to target. ATT&CK is useful for understanding security risk against known adversary behavior, for planning security improvements, and verifying defenses work as expected.

requirements.txt

@@ -1,17 +1,31 @@
 antlr4-python3-runtime==4.7.2
+cairocffi==1.1.0
+CairoSVG==2.4.2
 certifi==2018.11.29
+cffi==1.14.1
 chardet==3.0.4
+colour==0.1.5
+cssselect2==0.3.0
+defusedxml==0.6.0
+drawSvg==1.6.0
+et-xmlfile==1.0.1
 idna==2.8
+imageio==2.9.0
+jdcal==1.4.1
+numpy==1.19.1
+openpyxl==3.0.3
+Pillow==7.2.0
+pycparser==2.20
 python-dateutil==2.8.0
 pytz==2018.9
 requests==2.21.0
 simplejson==3.16.0
-six==1.12.0
-stix2==1.1.2
+six==1.15.0
+stix2==2.0.2
 stix2-patterns==1.1.0
 tabulate==0.8.3
-taxii2-client==0.5.0
+taxii2-client==2.2.1
+tinycss2==1.0.2
 tqdm==4.31.1
 urllib3==1.24.2
-colour==0.1.5
-openpyxl==3.0.3
+webencodings==0.5.1

scripts/diff_stix.py

@@ -1,6 +1,6 @@
 import argparse
 from stix2 import MemoryStore, Filter, TAXIICollectionSource
-from taxii2client import Collection
+from taxii2client.v20 import Collection
 import os
 import json
 from tqdm import tqdm

scripts/layers/samples/apt3_apt29_software.py

@@ -1,15 +1,15 @@
 import argparse
 import requests
 import json
-import stix2
+from stix2 import MemoryStore
 
 def generate(show_nodetect=False):
     """
         generate and return a layer dict showing techniques used by APT3 and APT29 as well as software used by those groups
         param show_nodetect, if true, causes techniques that have no data-sources to be highlighted as well
     """
     stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json()
-    ms = stix2.MemoryStore(stix_data=stix["objects"])
+    ms = MemoryStore(stix_data=stix["objects"])
     apt3 = ms.get("intrusion-set--0bbdf25b-30ff-4894-a1cd-49260d0dd2d9")
     apt29 = ms.get("intrusion-set--899ce53f-13a0-479b-a0e4-67d46e241542")
 

scripts/layers/samples/bear_APT.py

@@ -1,16 +1,16 @@
 import argparse
 import requests
 import json
-import stix2
+from stix2 import MemoryStore, Filter
 import re
 
 def generate():
     """parse the STIX on MITRE/CTI and return a layer dict showing all techniques used by an APT group with phrase 'bear' in the group aliases."""
     # import the STIX data from MITRE/CTI
     stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json()
-    ms = stix2.MemoryStore(stix_data=stix["objects"])
+    ms = MemoryStore(stix_data=stix["objects"])
 
-    groups = ms.query([ stix2.Filter("type", "=", "intrusion-set") ])
+    groups = ms.query([ Filter("type", "=", "intrusion-set") ])
 
     # find bear groups
     bear_groups = [] #list of groups with bear in name

scripts/layers/samples/heatmap.py

@@ -1,17 +1,17 @@
 import argparse
 import requests
 import json
-import stix2
+from stix2 import MemoryStore, Filter
 import random
 
 def generate():
     """parse the STIX on MITRE/CTI and return a layer dict with techniques with randomized scores"""
     # import the STIX data from MITRE/CTI
     stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json()
-    ms = stix2.MemoryStore(stix_data=stix["objects"])
+    ms = MemoryStore(stix_data=stix["objects"])
     # get all techniques in STIX
     techniques = ms.query([
-        stix2.Filter("type", "=", "attack-pattern")
+        Filter("type", "=", "attack-pattern")
     ])
     # parse techniques into layer format
     techniques_list = []

scripts/layers/samples/software_execution.py

@@ -1,7 +1,7 @@
 import argparse
 import requests
 import json
-import stix2
+from stix2 import Filter, MemoryStore
 from itertools import chain
 
 def generate(softwaretype="software"):
@@ -10,13 +10,13 @@ def generate(softwaretype="software"):
     """
     # import the STIX data from MITRE/CTI
     stix = requests.get("https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json").json()
-    ms = stix2.MemoryStore(stix_data=stix["objects"])
+    ms = MemoryStore(stix_data=stix["objects"])
     # software includes malware and tool types so perform two queries and merge the results
     software_filters = []
     if softwaretype == "malware" or softwaretype == "software":
-        software_filters.append( [ stix2.Filter('type', '=', 'malware') ] )
+        software_filters.append( [ Filter('type', '=', 'malware') ] )
     if softwaretype == "tool" or softwaretype == "software":
-        software_filters.append( [ stix2.Filter('type', '=', 'tool') ] )
+        software_filters.append( [ Filter('type', '=', 'tool') ] )
 
     software = list(chain.from_iterable(
         ms.query(f) for f in software_filters

scripts/technique_mappings_to_csv.py

@@ -2,8 +2,9 @@
 import csv
 import io
 
-import stix2
-import taxii2client
+from stix2 import TAXIICollectionSource, MemorySource, Filter
+from taxii2client.v20 import Collection
+
 import tqdm
 
 
@@ -15,18 +16,18 @@ def build_taxii_source(collection_name):
         "mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b"
     }
     collection_url = "https://cti-taxii.mitre.org/stix/collections/" + collection_map[collection_name] + "/"
-    collection = taxii2client.Collection(collection_url)
-    taxii_ds = stix2.TAXIICollectionSource(collection)
+    collection = Collection(collection_url)
+    taxii_ds = TAXIICollectionSource(collection)
 
     # Create an in-memory source (to prevent multiple web requests)
-    return stix2.MemorySource(stix_data=taxii_ds.query())
+    return MemorySource(stix_data=taxii_ds.query())
 
 
 def get_all_techniques(src, source_name):
     """Filters data source by attack-pattern which extracts all ATT&CK Techniques"""
     filters = [
-        stix2.Filter("type", "=", "attack-pattern"),
-        stix2.Filter("external_references.source_name", "=", source_name),
+        Filter("type", "=", "attack-pattern"),
+        Filter("external_references.source_name", "=", source_name),
     ]
     results = src.query(filters)
     return remove_deprecated(results)
@@ -35,13 +36,13 @@ def get_all_techniques(src, source_name):
 def filter_for_term_relationships(src, relationship_type, object_id, target=True):
     """Filters data source by type, relationship_type and source or target"""
     filters = [
-        stix2.Filter("type", "=", "relationship"),
-        stix2.Filter("relationship_type", "=", relationship_type),
+        Filter("type", "=", "relationship"),
+        Filter("relationship_type", "=", relationship_type),
     ]
     if target:
-        filters.append(stix2.Filter("target_ref", "=", object_id))
+        filters.append(Filter("target_ref", "=", object_id))
     else:
-        filters.append(stix2.Filter("source_ref", "=", object_id))
+        filters.append(Filter("source_ref", "=", object_id))
 
     results = src.query(filters)
     return remove_deprecated(results)
@@ -50,9 +51,9 @@ def filter_for_term_relationships(src, relationship_type, object_id, target=True
 def filter_by_type_and_id(src, object_type, object_id, source_name):
     """Filters data source by id and type"""
     filters = [
-        stix2.Filter("type", "=", object_type),
-        stix2.Filter("id", "=", object_id),
-        stix2.Filter("external_references.source_name", "=", source_name),
+        Filter("type", "=", object_type),
+        Filter("id", "=", object_id),
+        Filter("external_references.source_name", "=", source_name),
     ]
     results = src.query(filters)
     return remove_deprecated(results)

scripts/techniques_data_sources_vis.py

@@ -1,6 +1,6 @@
 import json, os, shutil, sys
 from stix2 import TAXIICollectionSource, Filter
-import taxii2client as t2c
+from taxii2client.v20 import Collection
 from pprint import pprint
 import argparse
 from tqdm import tqdm
@@ -94,19 +94,11 @@ def establish_connection(collection: str):
         connection to the taxii collection
     """
 
-    try:
-        with open('proxy.json', "r") as json_data:
-            proxies = json.load(json_data)
-        link = t2c._HTTPConnection()
-        link.session.proxies.update(proxies)
-        collection = t2c.Collection(collection, conn=link)
-        tc_src = TAXIICollectionSource(collection)
-    except:
-        if verbose: print("unable to find proxy file, falling back to standard callout... ", end="", flush=True)
-        # Establish TAXII2 Collection instance for Enterprise ATT&CK collection
-        collection = t2c.Collection(collection)
-        # Supply the collection to TAXIICollection
-        tc_src = TAXIICollectionSource(collection)
+
+    # Establish TAXII2 Collection instance for Enterprise ATT&CK collection
+    collection = Collection(collection)
+    # Supply the collection to TAXIICollection
+    tc_src = TAXIICollectionSource(collection)
     return tc_src
 
 

scripts/techniques_from_data_source.py

@@ -1,5 +1,5 @@
 from stix2 import TAXIICollectionSource, Filter
-from taxii2client import Collection
+from taxii2client.v20 import Collection
 import argparse
 
 # Establish TAXII2 Collection instance for Enterprise ATT&CK collection