diff --git a/external-import/group-ib/.env.sample b/external-import/group-ib/.env.sample new file mode 100644 index 0000000000..8425952def --- /dev/null +++ b/external-import/group-ib/.env.sample @@ -0,0 +1,48 @@ +# === OpenCTI Server === +# http://opencti:8080 - for docker deployment, http://localhost:8080 - for manual deployment +OPENCTI_URL=http://opencti:8080 +# Use one as provided by the OpenCTI platform +OPENCTI_TOKEN= + +# === OpenCTI Connector === +# Docker container name +CONTAINER_NAME=gib_connector +# Generate one with uuidgen (https://www.uuidgenerator.net/version4) +CONNECTOR_ID=80d059d5-7c3e-4b18-b307-2969a9461e6a +CONNECTOR_TYPE=EXTERNAL_IMPORT +# From 0 (Unknown) to 100 (Fully trusted) +CONNECTOR_CONFIDENCE_LEVEL=100 +# One of the following: debug, info, warning, error +CONNECTOR_LOG_LEVEL=debug +# the final letter should be one of 'd', 'h', 'm', 's' standing for day, hour, minute, second respectively. +CONNECTOR_RUN_EVERY=24h +CONNECTOR_UPDATE_EXISTING_DATA=true +CONNECTOR_SCOPE=stix2,ipv4-addr,ipv6-addr,vulnerability,domain,url,StixFile +CONNECTOR_NAME="Group-IB Connector" + +# === Threat Intelligence API Server === +# Connector specific parameters. All params are mandatory. +TI_API_URL=https://tap.group-ib.com/api/v2/ +TI_API_USERNAME= +TI_API_TOKEN= + +# === Threat Intelligence API Proxy === +# Proxy specifc parameters. You can leave them blank. +PROXY_IP= +PROXY_PORT= +PROXY_PROTOCOL= +PROXY_USERNAME= +PROXY_PASSWORD= + +# === Threat Intelligence Collections settings +# Collections specifc parameters. All params are mandatory. +IGNORE_NON_MALWARE_DDOS=true +IGNORE_NON_INDICATOR_THREAT_REPORTS=false + +# === RabbitMQ Server used for integration manual run === +MQ_HOST= +MQ_PORT= +MQ_VHOST= +MQ_USE_SSL= +MQ_USER= +MQ_PASS= diff --git a/external-import/group-ib/.gitignore b/external-import/group-ib/.gitignore new file mode 100644 index 0000000000..4dedc07637 --- /dev/null +++ b/external-import/group-ib/.gitignore @@ -0,0 +1,139 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ + +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +*.log.* +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + + +# Custom ignore +cache + +# Pycharm +.idea + +# Credentials diff --git a/external-import/group-ib/Dockerfile b/external-import/group-ib/Dockerfile new file mode 100644 index 0000000000..cd3059a051 --- /dev/null +++ b/external-import/group-ib/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11-alpine +ENV CONNECTOR_TYPE=EXTERNAL_IMPORT + +# Install Python modules +# hadolint ignore=DL3003 +RUN apk --no-cache add git build-base libmagic libffi-dev libxml2-dev libxslt-dev +COPY requirements.txt /tmp/requirements.txt +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt +RUN rm -rf /var/cache/apk/* + +# Copy the connector +COPY src /opt/connector/src +COPY docs /opt/connector/docs +WORKDIR /opt/connector/src + +# Install TI API lib +RUN pip3 install /opt/connector/src/lib/cyberintegrations-0.6.6-py3-none-any.whl + +# Expose and entrypoint +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/external-import/group-ib/README.md b/external-import/group-ib/README.md new file mode 100644 index 0000000000..0fb71cb72f --- /dev/null +++ b/external-import/group-ib/README.md @@ -0,0 +1,387 @@ +# OpenCTI Group-IB Connector + + +[![Python](https://img.shields.io/badge/python-v3.6.8+-blue?logo=python)](https://python.org/downloads/release/python-368/) +[![cyberintegrations](https://img.shields.io/badge/cyberintegrations-v0.6.6+-orange?)](https://github.com/cyberintegrations/releases/tag/0.6.6/) +[![OpenCTI](https://img.shields.io/badge/opencti-v6.2.0+-orange?)](https://github.com/OpenCTI-Platform/opencti/releases/tag/6.2.0) + + + + +The OpenCTI Group-IB Connector is a standalone Python process that collect data from Threat Intelligence via API calls +and push it as STIX objects to OpenCTI server. + +It is a system for cyber-attack analysis and attribution, threat hunting, and network infrastructure protection +based on data about adversary tactics, tools, and activities. TI combines unique data sources and experience in +investigating high-tech crimes and responding to complex, multi-stage attacks worldwide. The system stores data +on threat actors, domains, IPs, and infrastructure collected over the past 22 years, including those that criminals +have attempted to take down. + +To use the integration, please make sure that you have an active Threat Intelligence license to +access the interface. + + +## **Content** + +* [Content](#content) +* [Installation](#installation) + * [Common environment variables](#common-environment-variables) + * [OpenCTI environment variables](#opencti-environment-variables) + * [Threat Intelligence API environment variables](#threat-intelligence-api-environment-variables) + * [Docker Deployment](#docker-deployment) + * [Manual Deployment](#manual-deployment) +* [Configuration](#configuration) + * [Enable required collections](#enable-required-collections) + * [Date format](#date-format) + * [Notes](#notes) +* [Extra settings](#extra-settings) + * [Tags and options](#tags-and-options) +* [Examples](#examples) +* [Task scheduling automation](#task-scheduling-automation) + * [Cron](#cron) + * [Task Scheduler](#task-scheduler) +* [Troubleshooting](#troubleshooting) +* [FAQ](#faq) + * [Debugging](#debugging) + * [Additional information](#additional-information) + + + +
+ + + +## **Installation** + +### Requirements + +- Active Threat Intelligence license +- OpenCTI Platform >= 6.2.0 + + +### Common environment variables + +Configuration parameters are provided using environment variables as described below. +Some of them are placed directly in the `docker-compose.yml` since they are not expected to be modified by final +users once that they have been defined by the developer of the connector. + +Note that the values that follow can be grabbed within Python code using `self.helper.{PARAMETER}`, i. e., `self.helper.connector_name`. + +Expected environment variables to be set in the `docker-compose.yml` that describe the connector itself. +Most of the time, these values are NOT expected to be changed. + +| Parameter | Docker envvar | Mandatory | Description | +|--------------------------|-----------------------|-----------|-------------------------------------------------------------------| +| `connector_name` | `CONNECTOR_NAME` | Yes | A connector name to be shown in OpenCTI. | +| `connector_scope` | `CONNECTOR_SCOPE` | Yes | Supported scope. E. g., `text/html`. | +| `connector_id` | `CONNECTOR_ID` | Yes | A valid arbitrary `UUIDv4` that must be unique for this connector. | + +However, there are other values which are expected to be configured by end users. +The following values are expected to be defined in the `.env` file. +This file is included in the `.gitignore` (to avoid leaking sensitive data). +Note that the `.env.sample` file can be used as a reference. + +The ones that follow are connector's generic execution parameters expected to be added for export connectors. + +| Parameter | Docker envvar | Mandatory | Description | +|------------------------------|----------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `connector_confidence_level` | `CONNECTOR_CONFIDENCE_LEVEL` | Yes | The default confidence level for created sightings (a number between 1 and 4). | +| `connector_log_level` | `CONNECTOR_LOG_LEVEL` | Yes | The log level for this connector, could be `debug`, `info`, `warn` or `error` (less verbose). | +| `interval` | `CONNECTOR_RUN_EVERY` | Yes | The time unit is represented by a single character at the end of the string: d for days, h for hours, m for minutes, and s for seconds. e.g., 30s is 30 seconds. 1d is 1 day. | +| `update_existing_data` | `CONNECTOR_UPDATE_EXISTING_DATA` | Yes | Whether to update known existing data. | + + +### OpenCTI environment variables + +Below are the parameters you'll need to set for OpenCTI: + +| Parameter | Docker envvar | Mandatory | Description | +|-------------------------|-----------------------|-----------|------------------------------------------------------------------------------------------------------------------| +| `opencti_url` | `OPENCTI_URL` | Yes | The URL of the OpenCTI platform. Note that final `/` should be avoided. Example value: `http://opencti:8080` | +| `opencti_token` | `OPENCTI_TOKEN` | Yes | The default admin token configured in the OpenCTI platform parameters file. | + + +### Threat Intelligence API environment variables + +Below are the parameters you'll need to set for Threat Intelligence API: + +| Parameter | Docker envvar | Mandatory | Description | +|----------------------|---------------------------|-----------|------------------------------------------------| +| `ti_api_url` | `TI_API_URL` | Yes | Threat Intelligence API URL. | +| `ti_api_username` | `TI_API_USERNAME` | Yes | Threat Intelligence Portal profile email. | +| `ti_api_token` | `TI_API_TOKEN` | Yes | Threat Intelligence API Token. | + + +### Threat Intelligence API environment variables + +Below are the parameters you'll need to set if you have proxy server (if necessary): + +| Parameter | Docker envvar | Mandatory | Description | +|--------------------------|--------------------|-----------|-----------------| +| `proxy_ip` | `PROXY_IP` | No | Proxy IP. | +| `proxy_port` | `PROXY_PORT` | No | Proxy port. | +| `proxy_protocol` | `PROXY_PROTOCOL` | No | Proxy protocol. | +| `proxy_username` | `PROXY_USERNAME` | No | Proxy username. | +| `proxy_password` | `PROXY_PASSWORD` | No | Proxy password. | + + +### Docker Deployment + +Before building the Docker container, you need to set the version of pycti in `requirements.txt` +equal to whatever version of OpenCTI you're running. Example, `pycti==6.2.0`. If you don't, it will take +the latest version, but sometimes the OpenCTI SDK fails to initialize. + +Build a Docker Image using the provided `Dockerfile`. + +```bash +docker compose up -d +# -d for detached +``` + + +### Manual Deployment + +Install the required python dependencies (preferably in a virtual environment): + +```bash +pip3 install -r requirements.txt +``` + +Then, start the connector from `src`: + +```bash +python3 main.py +``` + + + +
+ + + +## Configuration + +Open ```docs/configs/endpoints_config.yaml``` file and fill in missing fields. +Before proceed, please check the [Starting Guide](https://tap.group-ib.com/hc/api?scope=integrations&q=en%2FIntegrations%2FStarting%20Guide%2FStarting%20Guide) +at our TI portal. + + +### Enable required collections + +The parameter ```default_date``` is used for initial start only. +After the download process begins it will not be used anymore. +Instead of ```default_date``` we will use ```seqUpdate``` parameter to iterate over the next portion. +It is technical field. +If you need fresh initial start based on the ```default_date```, please set ```seqUpdate``` parameter to ```null```. + +To start download process for any collection, you need to enable it first. +Set ```enable``` parameter to ```true```. +Set ```default_date``` parameter in single quotes, if needed or leave it as ```null```. +By default, it is set to 3 days back to the present time. + +```yaml + +collections: + attacks/ddos: + default_date: '2021-08-01' + enable: true + seqUpdate: null + attacks/phishing_group: + default_date: '2021-08-01' + enable: true + seqUpdate: null + ioc/common: + default_date: '2021-08-01' + enable: true + seqUpdate: 16747401659568 +... +``` + +### Date format + +Default date format. + +```'YYYY-MM-DD'``` + +### Notes + +*Note*: To use only IOCs (for example - Firewall rules), enable next collection: ```ioc/common```. + +*Note*: The ```ioc/common``` collection contains IoC only and based on ```malware/cnc```, +```hi/threat```, ```apt/threat```, ```hi/threat_actor```, ```apt/threat_actor``` collections. + +*Note*: ```attacks/deface```, ```attacks/ddos```, ```attacks/phishing_group```, ```suspicious_ip/open_proxy```, +```suspicious_ip/socks_proxy```, ```suspicious_ip/open_proxy```, ```suspicious_ip/tor_node```, +```suspicious_ip/vpn```, ```suspicious_ip/scanner``` - are very large collections, +and it is recommended to specify in the default_date field: 1-3 days ago. +Learn more about each collection +[here](https://tap.group-ib.com/hc/api?scope=integrations&q=en%2FIntegrations%2FDetailed%20collections%20info%2FDetailed%20collections%20info). + + +
+ + + +## Extra settings + + +### Tags and options + +In development + + + +
+ + + +## Examples + +Threat Reports + +![Reports](__docs__/media/reports.png) + +Threat Report with TI direct links + +![Report](__docs__/media/report.png) + +Threat Report `Knowledge` tab graph + +![Report graph](__docs__/media/report_graph.png) + +Indicators based on Observables + +![Indicators](__docs__/media/indicators.png) + +Threat Report Actors + +![Threat actors](__docs__/media/threat_actors.png) + +Threat Report Actor with related objects + +![Threat actor](__docs__/media/threat_actor.png) + +Threat Report Actor TTP + +![Threat actor TTP](__docs__/media/threat_actor_ttp.png) + +The way how relations names organized + +![mapping relationships](__docs__/media/mapping-relationships.png) + + +
+ + + +## **Task scheduling automation** + +Use operating system (cron or task scheduler) opportunities to automate daily +```main.py``` script execution. +You can use [Cron](https://crontab.guru/every-midnight) (for Unix-based systems) or Task Scheduler (for Windows). +The User must have appropriate rights. + +### Cron: + +- Open file **/etc/crontab** +- Set the following job: + +``` + +0 0 * * * cd %% && %%/python %%/main.py +``` + +- This job will start polling daily at midnight + +### Task Scheduler: + +- Create a new file **poller.bat** with the following content: + +``` + +“%%” “%%\main.py” +pause +``` + +- Go to **Control Panel** → Administrative Tools → Task Scheduler +- Choose the **Create Task** option. +- Fill in the task name and description. +- In the **Triggers** tab create a new daily trigger and set it to be repeated each 12 hours for an infinite amount of days. +- In the **Actions** tab add a new action and select the .bat file created on the first step. +- Click the **Ok** button. + + + +
+ + + +## Troubleshooting + +1. If you encounter problems, please retrieve logs from the **log** folder and attach them to +[Email](mailto:integration@group-ib.com) +or +[Service Desk](https://tap.group-ib.com/service_desk) +ticket. Also, please provide your TI portal email address and public IP address of integration app instance +(docker container IP / virtual machine IP) + +2. If you have problems with proxy configuration, attach the proxy environment by executing this command: +```printenv | grep proxy``` + + + +
+ + + +## FAQ + +1. Where I can find reports from last threats? + + They are separated similarly to TI interface. + **hi/threat** stands for Cybercriminals and **apt/threat** stands for Nation-State. + +2. What tags we are using for our events? + + Besides tags, that you can set in the configuration, we are using: + - collection name (but with spaces instead of "_" or "/") + - admiralty codes and TLP, where applicable + - for **osi/vulnerability** we add affected products as tags + - for **hi/threat** and **apt/threat** we add "Tailored" tag, similar to the TI portal + +3. Why app raise an access deni error in logs? + + Please check that OpenCTI user has correct rights and access. + Also, please check if you install integration app separate from the OpenCTI server virtual machine/docker container. + OpenCTI server is a separate app which control its folders rights. + + +### Debugging + +The connector can be debugged by setting the appropriate log level. +Note that logging messages can be added using `self.helper.log_{LOG_LEVEL}("Sample message")`, i. e., `self.helper.log_error("An error message")`. + + + +### Additional information + + + +If you face any errors with OpenCTI server images, remove existing docker images. +Warning, all your docker images will be deleted: + +```bash +rm -rf /var/lib/docker/* +``` \ No newline at end of file diff --git a/external-import/group-ib/__docs__/media/indicators.png b/external-import/group-ib/__docs__/media/indicators.png new file mode 100644 index 0000000000..ab9bec9f10 Binary files /dev/null and b/external-import/group-ib/__docs__/media/indicators.png differ diff --git a/external-import/group-ib/__docs__/media/mapping-relationships.png b/external-import/group-ib/__docs__/media/mapping-relationships.png new file mode 100644 index 0000000000..b15b13340d Binary files /dev/null and b/external-import/group-ib/__docs__/media/mapping-relationships.png differ diff --git a/external-import/group-ib/__docs__/media/report.png b/external-import/group-ib/__docs__/media/report.png new file mode 100644 index 0000000000..cf1be4f548 Binary files /dev/null and b/external-import/group-ib/__docs__/media/report.png differ diff --git a/external-import/group-ib/__docs__/media/report_graph.png b/external-import/group-ib/__docs__/media/report_graph.png new file mode 100644 index 0000000000..3a3a49488b Binary files /dev/null and b/external-import/group-ib/__docs__/media/report_graph.png differ diff --git a/external-import/group-ib/__docs__/media/reports.png b/external-import/group-ib/__docs__/media/reports.png new file mode 100644 index 0000000000..c48aaaa05a Binary files /dev/null and b/external-import/group-ib/__docs__/media/reports.png differ diff --git a/external-import/group-ib/__docs__/media/threat_actor.png b/external-import/group-ib/__docs__/media/threat_actor.png new file mode 100644 index 0000000000..e5c253fea3 Binary files /dev/null and b/external-import/group-ib/__docs__/media/threat_actor.png differ diff --git a/external-import/group-ib/__docs__/media/threat_actor_ttp.png b/external-import/group-ib/__docs__/media/threat_actor_ttp.png new file mode 100644 index 0000000000..d795bf0c50 Binary files /dev/null and b/external-import/group-ib/__docs__/media/threat_actor_ttp.png differ diff --git a/external-import/group-ib/__docs__/media/threat_actors.png b/external-import/group-ib/__docs__/media/threat_actors.png new file mode 100644 index 0000000000..2bccf28bfc Binary files /dev/null and b/external-import/group-ib/__docs__/media/threat_actors.png differ diff --git a/external-import/group-ib/docker-compose.yml b/external-import/group-ib/docker-compose.yml new file mode 100644 index 0000000000..b02be3261f --- /dev/null +++ b/external-import/group-ib/docker-compose.yml @@ -0,0 +1,33 @@ +version: '3' +services: + connector: + build: . + container_name: ${CONTAINER_NAME} + image: opencti/g-ti-connector:6.2.0 + environment: + # Connector's definition parameters: + - CONNECTOR_NAME=${CONNECTOR_NAME} + - CONNECTOR_SCOPE=${CONNECTOR_SCOPE} + # Connector's generic execution parameters: + - OPENCTI_URL=${OPENCTI_URL} + - OPENCTI_TOKEN=${OPENCTI_TOKEN} + - CONNECTOR_ID=${CONNECTOR_ID} + - CONNECTOR_CONFIDENCE_LEVEL=${CONNECTOR_CONFIDENCE_LEVEL} # From 0 (Unknown) to 100 (Fully trusted). + - CONNECTOR_LOG_LEVEL=${CONNECTOR_LOG_LEVEL} + - CONNECTOR_RUN_EVERY=${CONNECTOR_RUN_EVERY} + # Connector's TI API execution parameters: + - TI_API_URL=${TI_API_URL} + - TI_API_USERNAME=${TI_API_USERNAME} + - TI_API_TOKEN=${TI_API_TOKEN} + # Connector's proxy parameters: + - PROXY_IP=${PROXY_IP} + - PROXY_PORT=${PROXY_PORT} + - PROXY_PROTOCOL=${PROXY_PROTOCOL} + - PROXY_USERNAME=${PROXY_USERNAME} + - PROXY_PASSWORD=${PROXY_PASSWORD} + restart: always + +networks: + default: + external: true + name: docker_default diff --git a/external-import/group-ib/docs/configs/endpoints_config.yaml b/external-import/group-ib/docs/configs/endpoints_config.yaml new file mode 100644 index 0000000000..0e52e06cdc --- /dev/null +++ b/external-import/group-ib/docs/configs/endpoints_config.yaml @@ -0,0 +1,323 @@ +collections: + apt/threat: + default_date: null + description: A collection of Indicators and MITRE ATT&CK matrix. It contains HASH + sums of malicious files that were generated by hackers, IP addresses, domains, + CVE and the group's activities, motives, and goals to understand what tools + and tactics they use according to the MITRE ATT&CK matrix. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + apt/threat_actor: + default_date: null + description: Cybercriminal groups including nation-state (state-sponsored hacker + groups) and organized threat groups that target various industries and countries. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + attacks/ddos: + default_date: null + description: An attack that creates a load on the server and is executed simultaneously + from a large number of computers (often a network of infected zombie computers + is used) in order to create an artificial increase in requests to a resource + and thereby disable it. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + attacks/deface: + default_date: null + description: Defacement attacks are often conducted by web-hooligans (a form of + vandalism) or hacktivists (politically or religiously motivated actors) whose + aim is to draw attention to something. After a successful attack, the threat + actors publish information on special sites dedicated to defacement, social + media, or their personal sites. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + attacks/phishing_group: + default_date: null + description: The Phishing displays information about various phishing resources + (including sites masqueraded as Google, Microsoft, etc.). Group-IB collects + this data with the help of Passive-DNS analysis performed by Managed XDR (Managed + Extended Detection and Response) systems, alerts received by CERT-GIB, tracked + SPAM messages, malicious contextual advertising, new domain names, and other + valuable data. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + attacks/phishing_kit: + default_date: null + description: A Phishing kit is a collection of pages, scripts, and images that + keep a phishing website up and running. In other words, it is a ready-made phishing + website with a relevant settings file that specifies the parameters of how the + page needs to be displayed. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + compromised/access: + default_date: null + description: This collection displays the freshest information about compromised + data from various darkweb marketplaces (which sell illegal or restricted data + and services, according to the laws of a particular country). Most often it + is malware, hacked databases of social networks and so on. The information obtained + from this collection can help detect relevant threats that compromise company + employees, customers or systems on the internal network. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + compromised/account_group: + default_date: null + description: Hackers use phishing websites and malware for PC and Android to steal + logins and passwords. These can be credentials for the internal corporate systems + or external services for clients, such as Internet banking details. Malicious + programs transfer the intercepted data to attacker-controlled remote servers. + This server is the central data collection point for intercepted passwords and + other information that malware gathers. All data is distributed into the following + groups service, host, login and password. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + compromised/bank_card_group: + default_date: null + description: Bank Cards Group collection contains information about compromised + bank cards and masked cards. This includes data collected from card shops, specialized + forums, and public sources. All data collected is grouped by card number. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + compromised/discord: + default_date: null + description: The Discord collection contains data that was received by the TI + system from Discord. The Threat Intelligence system analyzes every chat and + channel (even private ones). Here detailed information about Discord servers, + channels and users can be extracted. You can also find data from the channels + which were added manually to the TI system. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: null + compromised/imei: + default_date: null + description: Android Trojans are designed to steal money from bank accounts, spy + on account holders, and extort money. They can intercept SMS messages, recover + passwords from cloud storage services, upload photo and video files, transmit + the device geolocation and lists of installed applications from a mobile device + to the threat actor, and automatically transfer funds. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + compromised/masked_card: + default_date: null + description: Masked Card collection contains information about compromised masked + cards. This includes data collected from card shops, specialized forums, and + public sources. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + compromised/messenger: + default_date: null + description: In this collection information from the Telegram chats and channels + can be found. The transferred or plan to transfer stolen money to. Man-in-the-Browser + (MITB) attacks, mobile Trojans, and phishing kits allow fraudsters to make money + transfers automatically. Analyzing bank-targeted botnets helps extract this + data from malware configuration files. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: null + compromised/mule: + default_date: null + description: This collection contains data about bank accounts threat actors have + transferred or plan to transfer stolen money to. Man-in-the-Browser (MITB) attacks, + mobile Trojans, and phishing kits allow fraudsters to make money transfers automatically. + Analyzing bank-targeted botnets helps extract this data from malware configuration + files. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + hi/open_threats: + default_date: null + description: The Open Threats collection consolidates public reports from various + cybersecurity vendors and researchers globally. All detected events are classified + by criteria such as threat actor, malware or country and tagged with common + identifiers. This makes it easier to understand the content at a glance and + apply intelligent filtering based on specific tags. The indicators from this + feed are automatically parsed, making it easy to integrate them into your security + measures. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: null + hi/threat: + default_date: null + description: A collection of Indicators and MITRE ATT&CK matrix. It contains HASH + sums of malicious files that were generated by hackers, IP addresses, domains, + CVE and group's activities, motives, and goals to understand what tools and + tactics they use according to the MITRE ATT&CK matrix. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + hi/threat_actor: + default_date: null + description: Cybercriminal groups including nation-state (state-sponsored hacker + groups) and organized threat groups that target various industries and countries. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + ioc/common: + default_date: null + description: The Common IoCs collection can help identify malicious activity or + security threats. Indicators of Compromise are clues and evidence of a data + breach, usually observed during a cybersecurity attack. Identified IoCs provide + the organization with a window into the techniques and methodologies of the + attackers who target them. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + malware/cnc: + default_date: null + description: Command and control. CNC collection contains information on the control + center where malware related to targeted attacks use to store stolen data or + download commands from. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 90 + malware/config: + default_date: null + description: Malicious files come from Malware control center. Contains HASH sums + of malicious files that were generated by hackers, IP addresses, and domains. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + malware/malware: + default_date: null + description: The Malware collection contains detailed information about specific + malware detected through analyzing Threat Actors activity. Can contain malware + names, related attacker names and additionally legitimate tools used by attackers + during an attack. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: null + malware/signature: + default_date: null + description: This collection contains malware signatures that can be used to enrich + malware security feeds, detect potentially confidential information and identify + specific malware promptly. Here the signature name, class and raw data can be + found (if detected). + enable: false + local_custom_tag: null + seqUpdate: null + ttl: null + malware/yara: + default_date: null + description: This collection includes data related to YARA rules and containing + information about specific malware families. Here YARA rule name, class and + raw data can be displayed (if detected). + enable: false + local_custom_tag: null + seqUpdate: null + ttl: null + osi/git_repository: + default_date: null + description: Open-source repositories such as GitHub contain codes that anyone + can search for. They are often used by threat actors planning to attack a specific + company. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + osi/public_leak: + default_date: null + description: There are specialized websites for exchanging textual information + (such as Pastebin and analogous resources). They can be used to upload texts + and send anyone a link to them. Both legitimate IT specialists and hackers actively + use such resources. IT professionals may underestimate the risks and load configuration + files for network equipment, export tables from databases, code fragments containing + access credentials, and much more. Hackers mainly post lists of usernames, passwords, + bank card details, Trojan configuration files, attack outcomes, and various + logs. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + osi/vulnerability: + default_date: null + description: The Vulnerability collection displays information about vulnerabilities + detected in the software by version. In addition to general information, the + subsection also contains data on existing exploits, with the option to view + links to PoC (Proof-of-Concept) and additional information, or to download the + exploit. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + suspicious_ip/open_proxy: + default_date: null + description: The Open proxy collection shows information about lists of proxy + servers that are publicly available on various Internet resources related to + anonymity. In addition, proxy servers may be configured as open proxies intentionally + or as a result of misconfiguration or breaches. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 15 + suspicious_ip/scanner: + default_date: null + description: This collection contains data about public and private IP-addresses + that were identified by the TI system. These records can be used to identify + or block connections between the corporate network and servers detected. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 15 + suspicious_ip/socks_proxy: + default_date: null + description: The Socks proxy collection shows information about addresses where + malware that turns infected computers into SOCKS proxies has been installed. + Such computers (bots) are rented out and used in various attacks to ensure the + attacker as much anonymity as possible. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 2 + suspicious_ip/tor_node: + default_date: null + description: The Tor collection displays data about Tor exit nodes, which are + the final Tor relays in the circuit. The nodes act as an intermediary between + a Tor client and public Internet. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 + suspicious_ip/vpn: + default_date: null + description: This collection contains information about public and private VPNs + servers that were identified by the TI system. These records can be used to + identify or block connections between the corporate network and servers detected. + enable: false + local_custom_tag: null + seqUpdate: null + ttl: 30 +extra_settings: + ignore_non_indicator_threats: false + ignore_non_malware_ddos: true + schedule_time: 00:00 + time_output_format: '%Y-%m-%d %H:%M:%S' diff --git a/external-import/group-ib/docs/configs/mapping.json b/external-import/group-ib/docs/configs/mapping.json new file mode 100644 index 0000000000..8f1265b2a3 --- /dev/null +++ b/external-import/group-ib/docs/configs/mapping.json @@ -0,0 +1,654 @@ +{ + "apt/threat": { + "threat_report": { + "__": "*Threat Report", + "id": "id", + "title": "title", + "portal_link": { + "__concatenate": { + "__": "TI Portal Report external reference", + "static": "https://tap.group-ib.com/ta/last-threats?threat=", + "dynamic": "id" + } + } + }, + "file": { + "__": "*IoC File", + "file_list": { + "__nested_dot_path_to_list": "indicators.params", + "md5": "hashes.md5", + "sha1": "hashes.sha1", + "sha256": "hashes.sha256", + "filename": "name", + "size-in-bytes": "size" + } + }, + "network": { + "__": "*IoC Network", + "network_list": { + "__nested_dot_path_to_list": "indicators.params", + "domain": "domain", + "url": "url", + "ip-address": "ipv4" + } + }, + "malware_report": { + "__": "*Malware Report", + "malware_report_list": { + "__nested_dot_path_to_list": "malwareList", + "name": "name", + "category": "category", + "platform": "platform", + "aliases": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Malware external reference", + "static": "https://tap.group-ib.com/malware/reports/", + "dynamic": "id" + } + } + } + }, + "threat_actor": { + "__": "*Threat Actor", + "name": "threatActor.name", + "country": "threatActor.country", + "targeted_countries": "countries", + "aliases": "", + "description": "", + "goals": "", + "roles": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Threat Actor external reference", + "static": "https://tap.group-ib.com/ta/actors?ta=", + "dynamic": "threatActor.id" + } + } + }, + "vulnerability": { + "__": "*List of CVE", + "vulnerability_list": { + "__nested_dot_path_to_list": "cveList", + "object_id": "name", + "description": "" + } + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "mitre_matrix": { + "__": "*MITRE ATT&CK", + "mitre_matrix_list": { + "__nested_dot_path_to_list": "mitreMatrix", + "attack_pattern": "mitreId", + "kill_chain_phase": "attackTactic", + "portal_link": { + "__concatenate": { + "__": "MITRE ATT&CK external reference", + "static": "https://attack.mitre.org/techniques/", + "dynamic": "mitreId" + } + } + } + }, + "date": { + "__": "*Event date", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen", + "date-published": "datePublished" + } + }, + "apt/threat_actor": { + "malware_report": { + "__": "*Malware Report", + "malware_report_list": { + "__nested_dot_path_to_list": "stat", + "name": "malware", + "category": "", + "platform": "", + "aliases": "", + "portal_link": {} + } + }, + "threat_actor": { + "__": "*Threat Actor", + "name": "name", + "country": "country", + "targeted_countries": "stat.countries", + "aliases": "aliases", + "description": "description", + "goals": "goals", + "roles": "roles", + "portal_link": { + "__concatenate": { + "__": "TI Portal Threat Actor external reference", + "static": "https://tap.group-ib.com/ta/actors?ta=", + "dynamic": "threatActor.id" + } + } + }, + "vulnerability": { + "__": "*List of CVE", + "vulnerability_list": { + "__nested_dot_path_to_list": "stat", + "object_id": "cve", + "description": "" + } + }, + "date": { + "__": "*Event date", + "first-seen": "stat.dateFirstSeen", + "last-seen": "stat.dateLastSeen" + } + }, + "attacks/ddos": { + "network": { + "__": "*IoC Network", + "domain": "cnc.domain", + "url": "cnc.url", + "ip-address": "cnc.ipv4.ip" + }, + "malware_report": { + "__": "*Malware Report", + "malware_report_list": { + "__nested_dot_path_to_list": "malware", + "name": "name", + "category": "category", + "platform": "platform", + "aliases": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Malware external reference", + "static": "https://tap.group-ib.com/malware/reports/", + "dynamic": "id" + } + } + } + }, + "threat_actor": { + "__": "*Threat Actor", + "name": "threatActor.name", + "country": "threatActor.country", + "targeted_countries": "", + "aliases": "", + "description": "", + "goals": "", + "roles": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Threat Actor external reference", + "static": "https://tap.group-ib.com/ta/actors?ta=", + "dynamic": "threatActor.id" + } + } + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "detection-date": "dateReg", + "submission-time": "dateBegin", + "takedown-time": "dateEnd" + } + }, + "attacks/deface": { + "network": { + "__": "*DEFACE Network", + "domain": "targetDomain", + "url": "url", + "ip-address": "targetIp.ip" + }, + "threat_actor": { + "__": "*Threat Actor", + "name": "threatActor.name", + "country": "threatActor.country", + "targeted_countries": "", + "aliases": "", + "description": "", + "goals": "", + "roles": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Threat Actor external reference", + "static": "https://tap.group-ib.com/ta/actors?ta=", + "dynamic": "threatActor.id" + } + } + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "detection-date": "date" + } + }, + "attacks/phishing_group": { + "network": { + "__": "*Phishing Network", + "network_list": { + "__nested_dot_path_to_list": "phishing", + "domain": "phishing_domain.domain", + "url": "url", + "ip-address": "phishing_ip.ip" + } + }, + "threat_actor": { + "__": "*Threat Actor", + "name": "threatActor.name", + "country": "", + "targeted_countries": "", + "aliases": "", + "description": "", + "goals": "", + "roles": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Threat Actor external reference", + "static": "https://tap.group-ib.com/ta/actors?ta=", + "dynamic": "threatActor.id" + } + } + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "submission-time": "date.detected", + "takedown-time": "date.blocked" + } + }, + "attacks/phishing_kit": { + "file": { + "__": "*Phishing File", + "md5": "hash", + "sha1": "hash", + "sha256": "hash", + "filename": "", + "size-in-bytes": "" + }, + "network": { + "__": "*Phishing source Network", + "network_list": { + "__nested_dot_path_to_list": "downloadedFrom", + "domain": "domain", + "url": "url", + "ip-address": "" + } + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "detection-date": "dateDetected", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen" + } + }, + "hi/threat": { + "threat_report": { + "__": "*Threat Report", + "id": "id", + "title": "title", + "portal_link": { + "__concatenate": { + "__": "TI Portal Report external reference", + "static": "https://tap.group-ib.com/ta/last-threats?threat=", + "dynamic": "id" + } + } + }, + "file": { + "__": "*IoC File", + "file_list": { + "__nested_dot_path_to_list": "indicators.params", + "md5": "hashes.md5", + "sha1": "hashes.sha1", + "sha256": "hashes.sha256", + "filename": "name", + "size-in-bytes": "size" + } + }, + "network": { + "__": "*IoC Network", + "network_list": { + "__nested_dot_path_to_list": "indicators.params", + "domain": "domain", + "url": "url", + "ip-address": "ipv4" + } + }, + "malware_report": { + "__": "*Malware Report", + "malware_report_list": { + "__nested_dot_path_to_list": "malwareList", + "name": "name", + "category": "category", + "platform": "platform", + "aliases": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Malware external reference", + "static": "https://tap.group-ib.com/malware/reports/", + "dynamic": "id" + } + } + } + }, + "threat_actor": { + "__": "*Threat Actor", + "name": "threatActor.name", + "country": "threatActor.country", + "targeted_countries": "countries", + "aliases": "", + "description": "", + "goals": "", + "roles": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Threat Actor external reference", + "static": "https://tap.group-ib.com/ta/actors?ta=", + "dynamic": "threatActor.id" + } + } + }, + "vulnerability": { + "__": "*List of CVE", + "vulnerability_list": { + "__nested_dot_path_to_list": "cveList", + "object_id": "name", + "description": "" + } + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "mitre_matrix": { + "__": "*MITRE ATT&CK", + "mitre_matrix_list": { + "__nested_dot_path_to_list": "mitreMatrix", + "attack_pattern": "mitreId", + "kill_chain_phase": "attackTactic", + "portal_link": { + "__concatenate": { + "__": "MITRE ATT&CK external reference", + "static": "https://attack.mitre.org/techniques/", + "dynamic": "mitreId" + } + } + } + }, + "date": { + "__": "*Event date", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen", + "date-published": "datePublished" + } + }, + "hi/threat_actor": { + "malware_report": { + "__": "*Malware Report", + "malware_report_list": { + "__nested_dot_path_to_list": "stat", + "name": "malware", + "category": "", + "platform": "", + "aliases": "", + "portal_link": {} + } + }, + "threat_actor": { + "__": "*Threat Actor", + "name": "name", + "country": "country", + "targeted_countries": "stat.countries", + "aliases": "aliases", + "description": "description", + "goals": "goals", + "roles": "roles", + "portal_link": { + "__concatenate": { + "__": "TI Portal Threat Actor external reference", + "static": "https://tap.group-ib.com/ta/actors?ta=", + "dynamic": "threatActor.id" + } + } + }, + "vulnerability": { + "__": "*List of CVE", + "vulnerability_list": { + "__nested_dot_path_to_list": "stat", + "object_id": "cve", + "description": "" + } + }, + "date": { + "__": "*Event date", + "first-seen": "stat.dateFirstSeen", + "last-seen": "stat.dateLastSeen" + } + }, + "malware/malware": { + "malware_report": { + "__": "*Malware Report", + "name": "name", + "category": "category", + "platform": "platform", + "aliases": "malwareAliasList", + "portal_link": { + "__concatenate": { + "__": "TI Portal Malware external reference", + "static": "https://tap.group-ib.com/malware/reports/", + "dynamic": "id" + } + } + }, + "date": { + "__": "*Event date", + "date-updated": "updatedAt" + } + }, + "malware/signature": { + "suricata_report": { + "sid": "sid", + "signature": "name", + "context": "content" + }, + "malware_report": { + "__": "*Malware Report", + "malware_report_list": { + "__nested_dot_path_to_list": "malware", + "name": "name", + "category": "", + "platform": "", + "aliases": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Malware external reference", + "static": "https://tap.group-ib.com/malware/reports/", + "dynamic": "id" + } + } + } + }, + "date": { + "__": "*Event date", + "date-created": "createdAt" + } + }, + "malware/yara": { + "yara_report": { + "yara": "name", + "yara-rule-name": "sourceName", + "context": "content" + }, + "malware_report": { + "__": "*Malware Report", + "malware_report_list": { + "__nested_dot_path_to_list": "malware", + "name": "name", + "category": "", + "platform": "", + "aliases": "", + "portal_link": { + "__concatenate": { + "__": "TI Portal Malware external reference", + "static": "https://tap.group-ib.com/malware/reports/", + "dynamic": "id" + } + } + } + }, + "date": { + "__": "*Event date", + "date-created": "createdAt" + } + }, + "osi/vulnerability": { + "vulnerability": { + "__": "*List of CVE", + "object_id": "id", + "description": "description" + }, + "cpe_table": { + "__description": "*CPE table for vulnerability", + "cpe_table_list": { + "__nested_dot_path_to_list": "cpeTable", + "type": "type", + "vendor": "vendor", + "product": "product", + "version": "version", + "string": "string", + "raw_string": "string23" + } + }, + "cvssv3": { + "__description": "*CVSSv3 information", + "score": "cvss.score", + "vector": "cvss.vector" + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "date-published": "datePublished", + "date-modified": "dateModified" + } + }, + "suspicious_ip/open_proxy": { + "network": { + "__": "*Open Proxy Network", + "ip-address": "ipv4.ip" + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen", + "detection-date": "dateDetected" + } + }, + "suspicious_ip/scanner": { + "network": { + "__": "*Scanner Network", + "ip-address": "ipv4.ip" + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen", + "detection-date": "dateDetected" + } + }, + "suspicious_ip/socks_proxy": { + "network": { + "__": "*Socks Proxy Network", + "ip-address": "ipv4.ip" + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen", + "detection-date": "dateDetected" + } + }, + "suspicious_ip/tor_node": { + "network": { + "__": "*Tor Node Network", + "ip-address": "ipv4.ip" + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen", + "detection-date": "dateDetected" + } + }, + "suspicious_ip/vpn": { + "network": { + "__": "*VPN Network", + "ip-address": "ipv4.ip" + }, + "evaluation": { + "__": "*Evaluation", + "tlp": "evaluation.tlp", + "admiralty_code": "evaluation.admiralty_code", + "severity": "evaluation.severity" + }, + "date": { + "__": "*Event date", + "first-seen": "dateFirstSeen", + "last-seen": "dateLastSeen", + "detection-date": "dateDetected" + } + } +} diff --git a/external-import/group-ib/docs/configs/qmapping.py b/external-import/group-ib/docs/configs/qmapping.py new file mode 100644 index 0000000000..bd529ba063 --- /dev/null +++ b/external-import/group-ib/docs/configs/qmapping.py @@ -0,0 +1,891 @@ +class TIAMapping: + """ + MISP-object: + "name" - is the name of your object. + "meta-category" - is the category where the object falls into. (such as file, network, financial, misc, internal...) + "description" - is a summary of the object description. + "version" - is the version number as a decimal value. + "required" - is an array containing the minimal required attributes to describe the object. + "requiredOneOf" - is an array containing the attributes where at least one needs to be present to describe the object. + "attributes" - key names which object expected + + Attributes nested info: + "misp-attribute" - field type + "ui-priority" - number, priority in order + "disable_correlation" - bool, create correlations or not. To suggest the disabling of correlation for a specific attribute. + "to_ids" - IDS (Intrusion Detection System) - is a system that monitors network traffic for suspicious activity and alerts when such activity is discovered. + + Malware/CnC: + Group by: + For groups: + - service.host + - login + - password + + For event: + - events[].cnc.cnc + - events[].source.id + - events[].source.type + - events[].clientIp.ip + - events[].malware.name + - events[].threatActor.name + + Contains CnC from: + - compromised/access + - compromised/account_group + - compromised/bank_card_group + - compromised/imei + - compromised/masked_card + - compromised/mule + + IoC/Common: + Contains: + - malware/cnc + - apt/threat + - hi/threat + - apt/threat_actor + - hi/threat_actor + + """ + + # MISP-object: ti-account + ACCOUNT = { + "__description": "*Compromised account information", + "service-domain": "service.domain", + "service-url": "service.url", + "username": "login", + "password": "password", + } + # MISP-object: ti-article-ip + ARTICLE_IP = { + "__description": "*IP addresses found in the article", + "ip-src": "data.ip.ip", + "port": "data.ip.port", + "name": "data.ip.tags", + } + # MISP-object: ti-article-domain + ARTICLE_DOMAIN = { + "__description": "*Domains found in the article", + "domain": "data.domains.value", + } + # MISP-object: ti-c2 + C2 = { + "__description": "*Command and Control Server (C2)", + "url": "cnc.url", + "domain": "cnc.domain", + "ip-address": "cnc.ipv4.ip", + "country": "cnc.ipv4.countryName", + "city": "cnc.ipv4.city", + "provider": "cnc.ipv4.provider", + "platform": "platform", + } + # MISP-object: ti-c2 + C2_INFO = { + "__description": "*Command and Control Server (C2) short", + "url": "url", + "domain": "domain", + "platform": "platform", + } + # MISP-object: ti-cpe-table + CPE_TABLE = { # Common Platform Enumeration + "__description": "*CPE table for vulnerability", # Common Vulnerabilities and Exposures CVE CCE? + "type": "cpeTable.type", + "vendor": "cpeTable.vendor", + "product": "cpeTable.product", + "version": "cpeTable.version", + } + # MISP-object: ti-cvssv2 + CVSSv2 = { # Common Vulnerability Scoring System + "__description": "*CVSSv2 information", + "score": "cvss.score", + "vector": "cvss.vector", + } + # MISP-object: ti-contacts + CONTACTS = {"__description": "*Contacts information", "contact": "contacts"} + # MISP-object: ti-chat + CHAT__TELEGRAM = { + "__description": "*Compromised chat information", + "server": "", + "chat": "name", + "chat-id": "chatStat.id", + "tittle": "chatStat.title", + "description": "", + "message": "message", + } + # MISP-object: ti-chat + CHAT__DISCORD = { + "__description": "*Compromised chat information", + "server": "channel.server", + "chat": "channel.name", + "chat-id": "channel.id", + "tittle": "", + "description": "channel.description", + "message": "text", + } + # MISP-object: ti-chat + CHAT_USER__TELEGRAM = { + "__description": "*Compromised chat user", + "user-id": "author.id", + "username": "author.userName", + "first-name": "author.firstName", + "last-name": "author.lastName", + } + # MISP-object: ti-chat-user + CHAT_USER__DISCORD = { + "__description": "*Compromised chat user", + "user-id": "author.id", + "username": "author.name", + "first-name": "", + "last-name": "", + } + # MISP-object: ti-chat-user + CHAT_USER__ARTICLE = { + "__description": "*Article author", + "user-id": "author.userId", + "username": "author.screenName", + "first-name": "author.name", + "last-name": "", + } + # MISP-object: ti-credit-card + CREDIT_CARD = { + "__description": "*Compromised card", + "cc-number": "cardInfo.number", + "card-security-code": "events.cardInfo.cvv", + "card-pin": "events.cardInfo.pin", + "bank_name": "cardInfo.issuer.issuer", + "payment-system": "cardInfo.system", + "type": "cardInfo.type", + "date": { + "expiration": "events.cardInfo.validThru", + "expiration-dt": "events.cardInfo.validThruDate", + }, + } + # MISP-object: ti-ddos + DDOS = { # MISP + "__description": "*DDOS activity information", + "text": "type", + "protocol": "protocol", + "dst-port": "port", + "ip-dst": "target.ipv4.ip", + "date": { + "first-seen": "dateBegin", + "last-seen": "dateEnd", + }, + } + # MISP-object: ti-email + EMAIL__ARTICLE = { + "__description": "*Emails found in the article", + "to": "data.emails", + } + # MISP-object: ti-file + FILE_IOC = { + "__description": "*IoC File", + "md5": "indicators.params.hashes.md5", + "sha1": "indicators.params.hashes.sha1", + "sha256": "indicators.params.hashes.sha256", + "filename": "indicators.params.name", + "size-in-bytes": "indicators.params.size", + } + # MISP-object: ti-file + FILE_IOC__HASH = {"__description": "*IoC File", "hash": "hash"} + # MISP-object: ti-file + FILE_IOC__ARTICLE = { + "__description": "*IoC File", + "md5": "data.files.hashes.md5", + "sha1": "data.files.hashes.sha1", + "sha256": "data.files.hashes.sha256", + "filename": "data.files.name", + "size-in-bytes": "data.files.size", + } + # MISP-object: ti-file + FILE_CONFIG = { + "__description": "*IoC File", + "md5": "*md5", + "sha1": "*sha1", + "sha256": "*sha256", + "filename": "*name", + "size-in-bytes": "*size", + } + # MISP-object: ti-leak-git-info + GIT_LEAK = { + "__description": "*Git repository leak information", + "repository": "name", + "match-type": "matchesTypes", + "date": {"detection-date": "dateDetected"}, + } + # MISP-object: ti-leak-git-revision + GIT_LEAK__REVISION = { + "__description": "*GIT repository leak revision information", + "file": "files.name", + "hash": "files.revisions.hash", + "author-name": "files.revisions.info.authorName", + "author-email": "files.revisions.info.authorEmail", + } + # MISP-object: ti-ip-address + # TODO: add extra info from ipv4 (provider, region) + IP_ADDRESS = { # MISP + "__description": "*Source IP address", + "ip-src": "ipv4.ip", + "country": "ipv4.countryName", + "country-code": "ipv4.countryCode", + "city": "ipv4.city", + "asn": "ipv4.asn", + "date": {"first-seen": "dateFirstSeen", "last-seen": "dateDetected"}, + } + # MISP-object: ti-ip-address + IP_ADDRESS__CLIENT = { # MISP + "__description": "*Source IP address", + "ip-src": "client.ipv4.ip", + "country": "client.ipv4.countryName", + "country-code": "client.ipv4.countryCode", + "city": "client.ipv4.city", + "asn": "client.ipv4.asn", + "date": {"first-seen": "dateFirstSeen", "last-seen": "dateDetected"}, + } + # MISP-object: ti-ip-address + IP_ADDRESS__CNC = { + "__description": "*Source IP address", + "ip-src": "cnc.ipv4.ip", + "country": "cnc.ipv4.countryName", + "country-code": "ipv4.countryCode", + "city": "cnc.ipv4.city", + "asn": "cnc.ipv4.asn", + "date": {"first-seen": "dateCompromised", "last-seen": "dateDetected"}, + } + # MISP-object: ti-ip-address + IP_ADDRESS__PHISHING = { + "__description": "*Phishing IP address information", + "ip-src": "phishing.phishing_ip.ip", + "country": "phishing.phishing_ip.country_name", + "country-code": "phishing.phishing_ip.country_code", + "city": "phishing.phishing_ip.city", + "provider": "phishing.phishing_ip.provider", + } + # MISP-object: ti-imei + IMEI = { + "__description": "*Compromised IMEI information", + "imei": "device.imei", + "model": "device.model", + "os": "device.os", + } + # MISP-object: ti-money-mule + MULE = { + "__description": "*Compromised mule information", + "account": "account", + "issuer": "organization.name", + } + # MISP-object: ti-malware + MALWARE = {"__description": "*Malware short information", "name": "malware.name"} + # MISP-object: ti-malware + MALWARE__LIST = { + "__description": "*Malware short information", + "name": "malwareList.name", + } + # MISP-object: ti-malware + MALWARE__ARTICLE = { + "__description": "*Malware short information", + "name": "data.malware.name", + } + # MISP-object: ti-network-profile + NETWORK_PROFILE = { + "__description": "*IoC Network", + "domain": "indicators.params.domain", + "url": "indicators.params.url", + "ip-address": "indicators.params.ipv4", + } + # MISP-object: ti-person + PERSON__OWNER = { + "__description": "*Personal information", + "address": "owner.address", + "full-name": "owner.name", + "e-mail": "owner.email", + "phone-number": "owner.phone", + } + # MISP-object: ti-person + PERSON__PERSON = { + "__description": "*Personal information", + "address": "person.address", + "full-name": "person.name", + "e-mail": "person.email", + "phone-number": "person.phone", + } + # MISP-object: ti-phishing + PHISHING = { # MISP + "__description": "*Phishing URL information", + "hostname": "phishingDomain.domain", + "url": "url", + "date": {"submission-time": "dateDetected", "takedown-time": "dateBlocked"}, + } + # MISP-object: ti-phishing + PHISHING__GROUP = { # MISP + "__description": "*Phishing URL information", + "domain": "phishing.phishing_domain.domain", + "url": "phishing.url", + "date": { + "submission-time": "phishing.date.detected", + "takedown-time": "phishing.date.blocked", + }, + } + # MISP-object: ti-file + PHISHING_KIT = {"__description": "*Phishing kit", "md5": "hash", "fullpath": "path"} + # MISP-object: ti-url + PHISHING_KIT_SOURCE = { + "__description": "*Phishing kit source", + "url": "downloadedFrom.url", + "domain": "downloadedFrom.domain", + } + # MISP-object: ti-email + PHISHING_KIT_EMAIL = { + "__description": "*Emails found in the phishing kit", + "to": "emails", + } + # MISP-object: ti-proxy-info + PROXY_INFO = { + "__description": "*Additional information about proxy", + "anonymous": "anonymous", + "port": "port", + "type": "type", + } + # MISP-object: gib-leak-public-content + PUBLIC_LEAK__CONTENT = { + "__description": "*Public leak information", + "author": "linkList.author", + "hash": "linkList.hash", + "link": "linkList.link", + "size": "linkList.size", + "source": "linkList.source", + "title": "linkList.title", + "date": { + "detection-date": "linkList.dateDetected", + "publishing-date": "linkList.datePublished", + }, + } + # MISP-object: gib-leak-public-info + PUBLIC_LEAK__INFO = { + "__description": "*Public leak common information", + "syntax": "language", + "hash": "hash", + "size": "size", + "creation-date": "created", + } + # MISP-object: gib-report + REPORT = { + "__description": "*Malware Report", + "report-id": "id", # Link: https://tap.group-ib.com/malware/reports/ + id + "name": "name", + "description": "shortDescription", + "platform": "platform", + "language": "langs", + "category": "category", + "malware-alias": "malwareAliasList", + "date": {"date-updated": "updatedAt"}, + } + # MISP-object: gib-sb-signature + SIGNATURE = { + "__description": "*Signature", + "sid": "sid", + "signature": "name", + "software": "malware.name", + "text": "content", + "date": {"date-created": "createdAt"}, + } + # MISP-object: gib-yara + YARA = { + "__description": "*Yara rule", + "yara": "name", + "yara-rule-name": "sourceName", + "context": "content", + } + # MISP-object: gib-threat + THREAT_INFO = { + "__description": "*Threat Actor information from threat Report", + "title": "title", + "country": "countries", + "language": "langs", + "region": "regions", + "sector": "sectors", + "source": "sources", + "date": {"date-published": "datePublished"}, + } + # MISP-object: gib-threat-actor + THREAT_ACTOR = { + "__description": "*Threat Actor short information", + "name": "threatActor.name", + } + # MISP-object: gib-threat-actor + THREAT_ACTOR__TA_LIST = { + "__description": "*Threat Actor short information", + "name": "taList.name", + } + # MISP-object: gib-threat-actor + THREAT_ACTOR__THREAT_ACTORS = { + "__description": "*Threat Actor short information", + "name": "threatActors.name", + } + # MISP-object: gib-threat-actor + THREAT_ACTOR__THREAT_LIST = { + "__description": "*Threat Actor short information", + "name": "threatList.name", + } + # MISP-object: gib-threat-actor + THREAT_ACTOR_INFO = { + "__description": "*Threat Actor information", + "name": "name", + "country": "country", + "description": "description", + "alias": "aliases", + "goal": "goals", + "language": "langs", + "role": "roles", + "date": {"first-seen": "dateFirstSeen", "last-seen": "dateLastSeen"}, + } + # MISP-object: gib-victim + VICTIM__NAME = {"__description": "*Phishing attack target", "name": "brand"} + # MISP-object: gib-victim + VICTIM__COMPANY = { + "__description": "*Targeted companies", + "name": "targetedCompany", + } + # MISP-object: gib-victim + VICTIM__TARGET = { + "__description": "*Targeted brand", + "name": "targetBrand", + "regions": "targetCountryName", + "sectors": "targetCategory", + "domain": "targetDomain", + } + # MISP-object: gib-victim-location + VICTIM_LOCATION__TARGET = { + "__description": "*The targeted victim information", + "url": "target.url", + "domain": "target.domain", + "ip-address": "target.ipv4.ip", + "country": "target.ipv4.countryName", + "city": "target.ipv4.city", + "provider": "target.ipv4.provider", + "category": "target.category", + } + # MISP-object: gib-victim-location + VICTIM_LOCATION__TARGET_IP = { + "__description": "*The targeted victim information", + "url": "url", + "domain": "targetDomain", + "ip-address": "targetIp.ip", + "country": "targetIp.countryName", + "city": "targetIp.city", + "provider": "targetIp.provider", + "category": "", + } + # MISP-object: gib-victim-location + VICTIM_LOCATION__CLIENT = { + "__description": "*The targeted victim information", + "url": "client.url", + "domain": "client.domain", + "ip-address": "client.ipv4.ip", + "country": "client.ipv4.countryName", + "city": "client.ipv4.city", + "provider": "client.ipv4.provider", + "category": "client.category", + } + # MISP-object: vulnerability + VULNERABILITY = { + "__description": "*List of CVEs connected to this threat", + "id": "cveList.name", + } + # MISP-object: vulnerability + VULNERABILITY__ARTICLE = { + "__description": "*List of CVEs connected to this article", + "id": "data.cve.id", + } + # MISP-object: vulnerability + VULNERABILITY_INFO = { + "__description": "*Vulnerability Information", + "id": "id", + "description": "extDescription", + "credit": "reporter", + "cvss-score": "extCvss.base", + "cvss-string": "extCvss.vector", + "vulnerable-configuration": "cpeTable.string23", + "references": "exploitList.href", + "date": {"published": "datePublished", "modified": "dateModified"}, + } + # MISP-object: gib-date + DATE = { + "__description": "*Event date", + "detection-date": "", + "publishing-date": "", + "first-seen": "", + "last-seen": "", + "expiration": "", + "expiration-dt": "", + "submission-time": "", + "takedown-time": "", + "date-created": "", + "date-updated": "", + "date-published": "", + "date-modified": "", + "date-compromised": "", + "date-add": "", + } + DATE__THREAT = { + "__description": "*Event date", + "date-created": "createdAt", # event created - no need for client(all) + "date-updated": "updatedAt", # event updated - no need for client(all) + "first-seen": "dateFirstSeen", # actor action first-seen + "last-seen": "dateLastSeen", # actor action last-seen + "date-published": "datePublished", # report published + } + DATE__THREAT_ACTOR = { + "__description": "*Event date", + "date-created": "createdAt", # event created + "date-updated": "updatedAt", # event updated + "first-seen": "stat.dateFirstSeen", # actor first-seen + "last-seen": "stat.dateLastSeen", # actor last-seen + } + DATE__DDOS = { + "__description": "*Event date", + "detection-date": "dateReg", # ddos detected + "submission-time": "dateBegin", # ddos start + "takedown-time": "dateEnd", # ddos end + } + DATE__DEFACE = { + "__description": "*Event date", + "detection-date": "date", # deface detected + } + DATE__PHISHING = { + "__description": "*Event date", + "date-created": "date.added", # event created + "date-updated": "date.updated", # event updated + "submission-time": "date.detected", # phishing detected + "takedown-time": "date.blocked", # phishing blocked + } + DATE__PHISHING_KIT = { + "__description": "*Event date", + "detection-date": "dateDetected", # phishing kit detected ? + "first-seen": "dateFirstSeen", # phishing kit changes first-seen ? + "last-seen": "dateLastSeen", # phishing kit changes last-seen ? + } + # MISP-object: gib-metadata + METADATA = { + "__description": "*Feed metadata", + "object-id": "id", + "portal-link": "portalLink", + "source": "source", + "source-url": "link", + "path-to-file": "path", + } + MITRE_MATRIX = { + "__description": "*MITRE Matrix Adversarial Tactics, Techniques & Common Knowledge", + "mitreMatrix": "mitreMatrix", # mitreId -> used by map from common/matrix/vocab/techniques + } + EVALUATION = { + "__description": "*Evaluation", + "severity": "evaluation.severity", # Severity level (green) + "tlp": "evaluation.tlp", # Traffic Light Protocol (amber) + "admiralty_code": "evaluation.admiraltyCode", # Data confidence level (C3) + } + + """ + TACTIC (Mitre) + FROM (Server) + HOW (DDOS) + WHO (Attacker) + WITH (Malware) + WHOM (Victim) + PAYLOAD (File) + ATTACK INFO (IP, Domain) + ATTACK DESCRIPTION (Vulnerability) + METADATA (Event metadata) + EVALUATION (TLP, Severity, Admiralty) + """ + MAPPING = { + # Collection apt/threat - Advance Persistence Threat + "apt/threat": { + "threat_actor": {**THREAT_ACTOR}, + "threat_info": {**THREAT_INFO}, + "malware__list": {**MALWARE__LIST}, + "victim_company": {**VICTIM__COMPANY}, + "file_ioc": {**FILE_IOC}, # Nested: indicators.params + "network_profile": {**NETWORK_PROFILE}, # Nested: indicators.params + "vulnerability": {**VULNERABILITY}, + "mitre_matrix": {**MITRE_MATRIX}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "apt/threat_actor": { + "threat_actor_info": {**THREAT_ACTOR_INFO}, + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + "attacks/ddos": { + "c2": {**C2}, + "ddos": {**DDOS}, + "threat_actor": {**THREAT_ACTOR}, + "malware": {**MALWARE}, + "victim_location": {**VICTIM_LOCATION__TARGET}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "attacks/deface": { + "threat_actor": {**THREAT_ACTOR}, + "victim_location": {**VICTIM_LOCATION__TARGET_IP}, + "contacts": {**CONTACTS}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "attacks/phishing": { + "phishing": {**PHISHING}, + "ip": {**IP_ADDRESS}, + "victim_brand": {**VICTIM__TARGET}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "attacks/phishing_group": { + "phishing": {**PHISHING__GROUP}, + "ip": {**IP_ADDRESS__PHISHING}, + "threat_actor": {**THREAT_ACTOR}, + "victim_brand": {**VICTIM__NAME}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "attacks/phishing_kit": { + "phishing_kit_source": {**PHISHING_KIT_SOURCE}, # Nested: downloadedFrom + "phishing_kit": {**PHISHING_KIT}, + "phishing_kit_email": {**PHISHING_KIT_EMAIL}, + "victim_brand": {**VICTIM__TARGET}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "compromised/access": { + "c2": {**C2}, + "ip_address_cnc": {**IP_ADDRESS__CNC}, + "malware": {**MALWARE}, + "victim_location": {**VICTIM_LOCATION__TARGET}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "compromised/account_group": { + "c2": {**C2}, # Nested: events + "threat_actor": {**THREAT_ACTOR}, + "malware": {**MALWARE}, + "account": {**ACCOUNT}, + "person": {**PERSON__PERSON}, # Nested: events + "victim_location": {**VICTIM_LOCATION__CLIENT}, # Nested: events + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "compromised/bank_card_group": { + "c2": {**C2}, # Nested: events + "threat_actor": {**THREAT_ACTOR}, + "malware": {**MALWARE}, + "ip_address__client": {**IP_ADDRESS__CLIENT}, # Nested: events + "person__owner": {**PERSON__OWNER}, # Nested: events + "credit_card": {**CREDIT_CARD}, # !!! - Need modifications + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "compromised/discord": { + "chat__discord": { + **CHAT__DISCORD + }, # chat - https://discordapp.com/channels/@me/userID/ 18 numb + "chat_user__discord": { + **CHAT_USER__DISCORD + }, # chat in server - https://discordapp.com/channels/serverID/chatID + # https://support.discord.com/hc/en-us/community/posts/360037884532-Link-to-enter-in-DM + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + # Collection compromised/imei - IMEI (International Mobile Equipment Identity) + "compromised/imei": { + "c2": {**C2}, + "threat_actor": {**THREAT_ACTOR}, + "malware": {**MALWARE}, + "imei": {**IMEI}, + "victim_location": {**VICTIM_LOCATION__CLIENT}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "compromised/masked_card": { + "c2": {**C2}, + "threat_actor": {**THREAT_ACTOR}, + "malware": {**MALWARE}, + "ip_address__client": {**IP_ADDRESS__CLIENT}, + "person__owner": {**PERSON__OWNER}, # !!! Need modifications + "credit_card": {**CREDIT_CARD}, # !!! Need modifications + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "compromised/messenger": { + "chat__telegram": {**CHAT__TELEGRAM}, # Link: https://t.me/ + chatStat.name + "chat_user__telegram": {**CHAT_USER__TELEGRAM}, + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + # Collection compromised/mule - All information is about the threat actor + "compromised/mule": { + "c2": {**C2}, + "threat_actor": {**THREAT_ACTOR}, + "malware": {**MALWARE}, + "mule": {**MULE}, + "person": {**PERSON__PERSON}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "hi/open_threats": { + "threat_actor__list": {**THREAT_ACTOR__THREAT_ACTORS}, + "malware__list": {**MALWARE__ARTICLE}, + "vulnerability": {**VULNERABILITY__ARTICLE}, + "file_ioc": {**FILE_IOC__ARTICLE}, + "emails": {**EMAIL__ARTICLE}, + "ip": {**ARTICLE_IP}, + "domain": {**ARTICLE_DOMAIN}, + "chat_user__article": {**CHAT_USER__ARTICLE}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "hi/threat": { + "threat_actor": {**THREAT_ACTOR}, + "threat_info": {**THREAT_INFO}, + "malware__list": {**MALWARE__LIST}, + "victim_company": {**VICTIM__COMPANY}, + "file_ioc": {**FILE_IOC}, # Nested: indicators.params + "network_profile": {**NETWORK_PROFILE}, # Nested: indicators.params + "vulnerability": {**VULNERABILITY}, + "mitre_matrix": {**MITRE_MATRIX}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "hi/threat_actor": { + "threat_actor_info": {**THREAT_ACTOR_INFO}, + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + "ioc/common": { + "threat_list": {**THREAT_ACTOR__THREAT_LIST}, + "malware__list": {**MALWARE__LIST}, + "file_ioc__hash": {**FILE_IOC__HASH}, + "network_profile": {**NETWORK_PROFILE}, + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + "malware/cnc": { + "c2_info": {**C2_INFO}, + "threat_actor": {**THREAT_ACTOR}, + "ip": {**IP_ADDRESS}, # Nested: ipv4 + "malware__list": {**MALWARE__LIST}, + "file_ioc": {**FILE_IOC}, # Nested: file + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + "malware/config": { + "malware": {**MALWARE}, + "file_config": {**FILE_CONFIG}, # Nested: file + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + # Collection malware/malware - Malware Report Description + "malware/malware": { + "report": {**REPORT}, + "threat_actor__list": {**THREAT_ACTOR__TA_LIST}, + "signature": {**SIGNATURE}, + "yara": {**YARA}, + "mitre_matrix": {**MITRE_MATRIX}, + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + "malware/signature": { + "signature": {**SIGNATURE}, + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + "malware/yara": { + "yara": {**YARA}, + "malware": {**MALWARE}, + "metadata": {**METADATA}, + # NO EVALUATION + "date": {**DATE}, + }, + "osi/git_repository": { + "git_leak": {**GIT_LEAK}, + "git_leak__revision": {**GIT_LEAK__REVISION}, # Nested: files + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "osi/public_leak": { + "public_leak__content": {**PUBLIC_LEAK__CONTENT}, # Nested: linkList + "public_leak__info": {**PUBLIC_LEAK__INFO}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "osi/vulnerability": { + "vulnerability_info": {**VULNERABILITY_INFO}, + "cpe_table": {**CPE_TABLE}, # Nested: cpeTable + "cvssv2": {**CVSSv2}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "suspicious_ip/open_proxy": { + "ip": {**IP_ADDRESS}, + "proxy_info": {**PROXY_INFO}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "suspicious_ip/scanner": { + "ip": {**IP_ADDRESS}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + # Collection suspicious_ip/socks_proxy - We know only IP not ports. Socks5 used with extra auth. More protected. + "suspicious_ip/socks_proxy": { + "ip": {**IP_ADDRESS}, + # P???_I??? + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "suspicious_ip/tor_node": { + "ip": {**IP_ADDRESS}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + "suspicious_ip/vpn": { + "ip": {**IP_ADDRESS}, + "metadata": {**METADATA}, + "evaluation": {**EVALUATION}, + "date": {**DATE}, + }, + } + + +# x = TIAMapping() +# import json +# print(json.dumps(x.MAPPING, indent=4)) diff --git a/external-import/group-ib/entrypoint.sh b/external-import/group-ib/entrypoint.sh new file mode 100644 index 0000000000..77f91c125a --- /dev/null +++ b/external-import/group-ib/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# Start the connector (WORKDIR is /opt/connector as set in the Dockerfile) +python3 main.py diff --git a/external-import/group-ib/requirements.txt b/external-import/group-ib/requirements.txt new file mode 100644 index 0000000000..650bd6e355 --- /dev/null +++ b/external-import/group-ib/requirements.txt @@ -0,0 +1,3 @@ +pycti==6.2.0 +python-dotenv==1.0.1 +stix2==3.0.1 \ No newline at end of file diff --git a/external-import/group-ib/src/adapter.py b/external-import/group-ib/src/adapter.py new file mode 100644 index 0000000000..df33418161 --- /dev/null +++ b/external-import/group-ib/src/adapter.py @@ -0,0 +1,935 @@ +from datetime import datetime + +from stix2.patterns import HashConstant + +import data_to_stix2 as ds + + +class DataToSTIXAdapter: + + def __init__(self, mitre_mapper, collection, tlp_color, helper, is_ioc=False): + # type: (dict, str, str, Any, bool) -> None + self.mitre_mapper = mitre_mapper + self.collection = collection + self.ta_global_label = self._set_global_label(self.collection) + self.tlp_color = tlp_color + self.is_ioc = is_ioc + self.helper = helper + + @staticmethod + def _valid_hash(hash_value, hash_type): + try: + HashConstant(value=hash_value, type=hash_type) + return True + except ValueError: + return False + + def _set_global_label(self, collection): + if collection in ["apt/threat", "apt/threat_actor"]: + return "nation-state" + elif collection in ["hi/threat", "hi/threat_actor"]: + return "criminal" + + @staticmethod + def _retrieve_link(obj): + # type: (Union[dict, list]) -> List[Tuple[str, str, str]] + if isinstance(obj, list): + result = list() + for _o in obj: + _link = _o.get("portal_link", {}) + if _link: + link_id = _link.get("dynamic") + link_url = _link.get("result") + link_description = _link.get("__") + result.append((link_id, link_url, link_description)) + return result + else: + result = list() + _link = obj.get("portal_link", {}) + if _link: + link_id = _link.get("dynamic") + link_url = _link.get("result") + link_description = _link.get("__") + result.append((link_id, link_url, link_description)) + return result + + @staticmethod + def _generate_relations(main_obj, related_objects, relation_type=None): + # type: (Any, List[Any], Union[None, str]) -> Any + + # TODO: create common relationship map for all objects + relation_type_map = { + "threat_actor": { + "attack_pattern": "uses", + "malware": "uses", + "vulnerability": "targets", + "file": "related-to", + "base_location": "located-at", + "target_location": "targets", + }, + "indicator": {"file": "based-on", "ipv4": "based-on", "ipv6": "based-on"}, + "ipv4": { + "threat_actor": "related-to", + "domain": "related-to", + "url": "related-to", + }, + "ipv6": { + "threat_actor": "related-to", + "domain": "related-to", + "url": "related-to", + }, + "domain": {"threat_actor": "related-to"}, + "url": {"threat_actor": "related-to"}, + "vulnerability": {}, + "malware": {"ipv4": "communicates-with", "ipv6": "communicates-with"}, + } + # relation_type_map = { + # "attack_pattern": "uses", + # "malware": "uses", + # "vulnerability": "targets" + # } + relation_type_required = True + if relation_type: + relation_type_required = False + + for _rel_obj in related_objects: + if _rel_obj: + if isinstance(_rel_obj, list) and _rel_obj: + for _ro in _rel_obj: + if relation_type_required: + relation_type = relation_type_map.get(main_obj.type).get( + _ro.type, None + ) + if not relation_type: + raise AttributeError + main_obj.generate_relationship( + main_obj.stix_main_object, + _ro.stix_main_object, + relation_type=relation_type, + ) + else: + + if relation_type_required: + relation_type = relation_type_map.get(main_obj.type).get( + _rel_obj.type, None + ) + if not relation_type: + raise AttributeError + main_obj.generate_relationship( + main_obj.stix_main_object, + _rel_obj.stix_main_object, + relation_type=relation_type, + ) + return main_obj + + def _generate_mitre_matrix(self, obj_events): + mitre_matrix = { + _e.get("attack_pattern"): { + "kill_chain_phases": list(), + "portal_links": list(), + } + for _e in obj_events + if _e.get("attack_pattern") + } + for _e in obj_events: + if _e.get("attack_pattern"): + mitre_matrix[_e.get("attack_pattern")]["kill_chain_phases"].append( + _e.get("kill_chain_phase") + ) + mitre_matrix[_e.get("attack_pattern")]["portal_links"] = ( + self._retrieve_link(_e) + ) + return mitre_matrix + + def generate_kill_chain_phases(self, obj_types): + _name = "mitre-attack" + _label = "mitre" + + return [ + ds.KillChainPhase( + name=_name, + _type=_type, + # tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + .generate_stix_objects() + .stix_main_object + for _type in obj_types + ] + + def generate_stix_domain(self, name): + _type = "domain" + _label = "domain" + + return ds.Domain( + name=name, + _type=_type, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + + def generate_stix_url(self, name): + _type = "url" + _label = "url" + + return ds.URL( + name=name, + _type=_type, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + + def generate_stix_ipv4(self, name): + _type = "ipv4" + _label = "ipv4" + + return ds.IPAddress( + name=name, + _type=_type, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + + def generate_locations(self, obj_country_codes): + _type = "location" + _label = "country" + + return [ + ds.Location( + name=_cc, + _type=_type, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ).generate_stix_objects() + for _cc in obj_country_codes + if _cc + ] + + def generate_stix_malware(self, obj, json_date_obj=None): + if not obj: + return list() + + # _description = obj.get("__") + _type = "malware" + _label = "malware" + _events = obj.get("malware_report_list", []) + _date_updated = None + + if json_date_obj: + try: + _date_updated = datetime.strptime( + json_date_obj.get("date-updated"), "%Y-%m-%dT%H:%M:%S%z" + ) + except (Exception,): + self.helper.log_warning( + "Failed to format first_seen: {}. Using default.".format( + json_date_obj.get("date-updated"), + ) + ) + _date_updated = None + + _stix_objects = list() + + if _events: + for _e in _events: + _name = _e.get("name") + _malware_types = _e.get("category") + _malware_aliases = _e.get("aliases") + + _portal_links = self._retrieve_link(_e) + + if _name: + if isinstance(_name, list): + for n in _name: + malware = ds.Malware( + name=n, + aliases=_malware_aliases, + last_seen=_date_updated, + _type=_type, + malware_types=_malware_types or [], + tlp_color="red", + labels=[self.collection, _label], + ) + malware.generate_external_references(_portal_links) + malware.generate_stix_objects() + + _stix_objects.append(malware) + else: + malware = ds.Malware( + name=_name, + aliases=_malware_aliases, + last_seen=_date_updated, + _type=_type, + malware_types=_malware_types, + tlp_color="red", + labels=[self.collection, _type], + ) + malware.generate_external_references(_portal_links) + malware.generate_stix_objects() + + _stix_objects.append(malware) + + else: + _name = obj.get("name") + _malware_types = obj.get("category") + _malware_aliases = obj.get("aliases") + + _portal_links = self._retrieve_link(obj) + + if _name: + malware = ds.Malware( + name=_name, + aliases=_malware_aliases, + last_seen=_date_updated, + _type=_type, + malware_types=_malware_types, + tlp_color="red", + labels=[self.collection, _type], + ) + malware.generate_external_references(_portal_links) + malware.generate_stix_objects() + + _stix_objects.append(malware) + + return _stix_objects + + def generate_stix_vulnerability( + self, obj, related_objects, json_date_obj=None, json_cvss_obj=None + ): + if not obj: + return list() + + # TODO: How to add cvssv2??? cpeTable? + + _description = obj.get("__") + _type = "vulnerability" + _label = "vulnerability" + if json_cvss_obj: + _cvssv3_score = json_cvss_obj.get("score", None) + _cvssv3_vector = json_cvss_obj.get("vector", None) + else: + _cvssv3_score = None + _cvssv3_vector = None + _events = obj.get("vulnerability_list", []) + _date_published = None + + if json_date_obj: + try: + _date_published = datetime.strptime( + json_date_obj.get("date-published"), "%Y-%m-%dT%H:%M:%S%z" + ) + except (Exception,): + self.helper.log_warning( + "Failed to format first_seen: {}. Using default.".format( + json_date_obj.get("date-updated"), + ) + ) + _date_published = datetime.now() + + _stix_objects = list() + + if _events: + for _e in _events: + _name = _e.get("object_id") + _description = _e.get("description") + + if _name: + if isinstance(_name, list): + for n in _name: + vulnerability = ds.Vulnerability( + name=n, + _type=_type, + created=_date_published, + cvss_score=_cvssv3_score, + cvss_vector=_cvssv3_vector, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + vulnerability.generate_stix_objects() + + if related_objects: + self._generate_relations( + vulnerability, related_objects, "related-to" + ) + + vulnerability.set_description(_description) + vulnerability.add_relationships_to_stix_objects() + + _stix_objects.append(vulnerability) + else: + vulnerability = ds.Vulnerability( + name=_name, + _type=_type, + created=_date_published, + cvss_score=_cvssv3_score, + cvss_vector=_cvssv3_vector, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + vulnerability.generate_stix_objects() + + if related_objects: + self._generate_relations( + vulnerability, related_objects, "related-to" + ) + + vulnerability.set_description(_description) + vulnerability.add_relationships_to_stix_objects() + + _stix_objects.append(vulnerability) + + else: + _name = obj.get("object_id") + _description = obj.get("description") + + if _name: + vulnerability = ds.Vulnerability( + name=_name, + _type=_type, + created=_date_published, + cvss_score=_cvssv3_score, + cvss_vector=_cvssv3_vector, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + vulnerability.generate_stix_objects() + + if related_objects: + self._generate_relations( + vulnerability, related_objects, "related-to" + ) + + vulnerability.set_description(_description) + vulnerability.add_relationships_to_stix_objects() + + _stix_objects.append(vulnerability) + + return _stix_objects + + def generate_stix_attack_pattern(self, obj): + if not obj: + return list() + + _description = obj.get("__") + _type = "attack_pattern" + _label = "attack_pattern" + _events = obj.get("mitre_matrix_list") + + _stix_objects = list() + + event_mitre_matrix = self._generate_mitre_matrix(_events) + + for k, v in event_mitre_matrix.items(): + + kill_chain_phases = self.generate_kill_chain_phases(v["kill_chain_phases"]) + + if k: + attack_pattern = ds.AttackPattern( + name=self.mitre_mapper.get(k), + _type=_type, + mitre_id=k, + kill_chain_phases=kill_chain_phases, + # tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + attack_pattern.set_description(_description) + attack_pattern.generate_external_references(v["portal_links"]) + attack_pattern.generate_stix_objects() + + _stix_objects.append(attack_pattern) + + return _stix_objects + + def generate_stix_threat_actor(self, obj, related_objects, json_date_obj=None): + if not obj: + return None, None + + # _description = obj.get("__") + _type = "threat_actor" + _label = "threat_actor" + _global_label = self.ta_global_label + # _country_type = "country" + + _threat_actor_name = obj.get("name") + _threat_actor_country = obj.get("country") + _threat_actor_targeted_countries = obj.get("targeted_countries") + _threat_actor_aliases = obj.get("aliases") + _threat_actor_description = obj.get("description") + _threat_actor_goals = obj.get("goals") + _threat_actor_roles = obj.get("roles") + _date_first_seen = None + _date_last_seen = None + + if json_date_obj: + try: + _date_first_seen = datetime.strptime( + json_date_obj.get("first-seen"), "%Y-%m-%d" + ) + _date_last_seen = datetime.strptime( + json_date_obj.get("last-seen"), "%Y-%m-%d" + ) + except (Exception,): + self.helper.log_warning( + "Failed to format first_seen: {}, last_seen: {}. Using default.".format( + json_date_obj.get("first-seen"), json_date_obj.get("last-seen") + ) + ) + _date_first_seen = None + _date_last_seen = None + + _portal_link = self._retrieve_link(obj) + + threat_actor = None + locations = None + + if _threat_actor_name: + threat_actor = ds.ThreatActor( + name=_threat_actor_name, + _type=_type, + global_label=_global_label, + tlp_color="red", + labels=[self.collection, _label], + aliases=_threat_actor_aliases, + first_seen=_date_first_seen, + last_seen=_date_last_seen, + goals=_threat_actor_goals, + roles=_threat_actor_roles, + ) + threat_actor.set_description(_threat_actor_description) + threat_actor.generate_external_references(_portal_link) + threat_actor.generate_stix_objects() + + base_locations = [] + if _threat_actor_country: + base_locations = self.generate_locations([_threat_actor_country]) + target_locations = [] + if _threat_actor_targeted_countries: + target_locations = self.generate_locations( + _threat_actor_targeted_countries + ) + + locations = base_locations + target_locations + + if _threat_actor_name and base_locations: + self._generate_relations(threat_actor, base_locations, "located-at") + + if _threat_actor_name and target_locations: + self._generate_relations(threat_actor, target_locations, "targets") + + if related_objects: + self._generate_relations(threat_actor, related_objects) + + threat_actor.add_relationships_to_stix_objects() + + return threat_actor, locations + + def generate_stix_file(self, obj, related_objects=None, is_ioc=True): + if not obj: + return list() + + _description = obj.get("__") + _type = "file" + _label = "file" + _events = obj.get("file_list") + + _stix_objects = list() + + if _events: + for _e in _events: + _md5 = _e.get("md5", None) + _sha1 = _e.get("sha1", None) + _sha256 = _e.get("sha256", None) + if _md5: + if not self._valid_hash(_md5, "MD5"): + self.helper.log_error( + f"Error! {_md5} is not valid MD5. Ignored." + ) + _md5 = None + if _sha1: + if not self._valid_hash(_sha1, "SHA1"): + self.helper.log_error( + f"Error! {_sha1} is not valid SHA1. Ignored." + ) + _sha1 = None + if _sha256: + if not self._valid_hash(_sha256, "SHA256"): + self.helper.log_error( + f"Error! {_sha256} is not valid SHA256. Ignored." + ) + _sha256 = None + hashes = [_md5, _sha1, _sha256] + + if any(hashes): + file = ds.FileHash( + name=hashes, + _type=_type, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + file.set_description(_description) + file.is_ioc = is_ioc + file.generate_stix_objects() + + if self.is_ioc: + for ind in file.stix_indicator: + file.generate_relationship( + ind, file.stix_main_object, relation_type="based-on" + ) + + if related_objects: + self._generate_relations(file, related_objects, "related-to") + + file.add_relationships_to_stix_objects() + + _stix_objects.append(file) + else: + _md5 = obj.get("md5", None) + _sha1 = obj.get("sha1", None) + _sha256 = obj.get("sha256", None) + if _md5: + if not self._valid_hash(_md5, "MD5"): + self.helper.log_error(f"Error! {_md5} is not valid MD5. Ignored.") + _md5 = None + if _sha1: + if not self._valid_hash(_sha1, "SHA1"): + self.helper.log_error(f"Error! {_sha1} is not valid SHA1. Ignored.") + _sha1 = None + if _sha256: + if not self._valid_hash(_sha256, "SHA256"): + self.helper.log_error( + f"Error! {_sha256} is not valid SHA256. Ignored." + ) + _sha256 = None + hashes = [_md5, _sha1, _sha256] + + if any(hashes): + file = ds.FileHash( + name=hashes, + _type=_type, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + file.set_description(_description) + file.is_ioc = is_ioc + file.generate_stix_objects() + + if self.is_ioc: + for ind in file.stix_indicator: + file.generate_relationship( + ind, file.stix_main_object, relation_type="based-on" + ) + + if related_objects: + self._generate_relations(file, related_objects, "related-to") + + file.add_relationships_to_stix_objects() + + _stix_objects.append(file) + + return _stix_objects + + def generate_stix_network( + self, + obj, + related_objects=None, + url_is_ioc=False, + domain_is_ioc=False, + ip_is_ioc=False, + ): + if not obj: + return list(), list(), list() + + _description = obj.get("__") + # _type = "ipv4" + # _label = "ipv4" + _events = obj.get("network_list", None) + + _domain_stix_objects = list() + _url_stix_objects = list() + _ip_stix_objects = list() + + if _events: + for _e in _events: + _domain = _e.get("domain") + _url = _e.get("url") + _ips = _e.get("ip-address") + + domain = None + if _domain: + domain = self.generate_stix_domain(_domain) + domain.is_ioc = domain_is_ioc + domain.generate_stix_objects() + + if domain_is_ioc: + domain.generate_relationship( + domain.stix_indicator, + domain.stix_main_object, + relation_type="based-on", + ) + + if related_objects: + self._generate_relations(domain, related_objects, "related-to") + + domain.add_relationships_to_stix_objects() + + _domain_stix_objects.append(domain) + + url = None + if _url: + url = self.generate_stix_url(_url) + url.is_ioc = url_is_ioc + link_id = "" + link_url = _url + link_description = "Source external reference" + url.generate_external_references( + [(link_id, link_url, link_description)] + ) + url.generate_stix_objects() + + if url_is_ioc: + url.generate_relationship( + url.stix_indicator, + url.stix_main_object, + relation_type="based-on", + ) + + if related_objects: + self._generate_relations(url, related_objects, "related-to") + + url.add_relationships_to_stix_objects() + + _url_stix_objects.append(url) + + if _ips: + for _ip in _ips: + ip = self.generate_stix_ipv4(_ip) + + ip.set_description(_description) + ip.is_ioc = ip_is_ioc + ip.generate_stix_objects() + + if ip_is_ioc: + ip.generate_relationship( + ip.stix_indicator, + ip.stix_main_object, + relation_type="based-on", + ) + + if related_objects: + self._generate_relations(ip, related_objects, "related-to") + + if domain: + self._generate_relations(ip, [domain], "related-to") + if url: + self._generate_relations(ip, [url], "related-to") + + ip.add_relationships_to_stix_objects() + + _ip_stix_objects.append(ip) + + else: + _domain = obj.get("domain") + _url = obj.get("url") + _ip = obj.get("ip-address") + + domain = None + if _domain: + domain = self.generate_stix_domain(_domain) + domain.is_ioc = domain_is_ioc + domain.generate_stix_objects() + + if domain_is_ioc: + domain.generate_relationship( + domain.stix_indicator, + domain.stix_main_object, + relation_type="based-on", + ) + + if related_objects: + self._generate_relations(domain, related_objects, "related-to") + + domain.add_relationships_to_stix_objects() + + _domain_stix_objects.append(domain) + + url = None + if _url: + url = self.generate_stix_url(_url) + url.is_ioc = url_is_ioc + link_id = "" + link_url = _url + link_description = "Source external reference" + url.generate_external_references( + [(link_id, link_url, link_description)] + ) + url.generate_stix_objects() + + if url_is_ioc: + url.generate_relationship( + url.stix_indicator, + url.stix_main_object, + relation_type="based-on", + ) + + if related_objects: + self._generate_relations(url, related_objects, "related-to") + + url.add_relationships_to_stix_objects() + + _url_stix_objects.append(url) + + if _ip: + ip = self.generate_stix_ipv4(_ip) + + ip.set_description(_description) + ip.is_ioc = ip_is_ioc + ip.generate_stix_objects() + + if ip_is_ioc: + ip.generate_relationship( + ip.stix_indicator, ip.stix_main_object, relation_type="based-on" + ) + + if related_objects: + self._generate_relations(ip, related_objects, "related-to") + + if domain: + self._generate_relations(ip, [domain], "related-to") + if url: + self._generate_relations(ip, [url], "related-to") + + ip.add_relationships_to_stix_objects() + + _ip_stix_objects.append(ip) + + return _domain_stix_objects, _url_stix_objects, _ip_stix_objects + + def generate_stix_report( + self, + obj, + json_date_obj, + report_related_objects_ids, + json_malware_report_obj, + json_threat_actor_obj, + ): + if not obj: + return None + + _description = obj.get("title") + _type = "threat_report" + _label = "threat_report" + _id = obj.get("id") + _date_published = json_date_obj.get("date-published") + + _report_portal_links = self._retrieve_link(obj) + _threat_actor_portal_links = self._retrieve_link(json_threat_actor_obj) + _malware_portal_links = self._retrieve_link( + json_malware_report_obj.get("malware_report_list") + ) + report_links = ( + _report_portal_links + _threat_actor_portal_links + _malware_portal_links + ) + + ta_label = json_threat_actor_obj.get("name") + + report = ds.Report( + name=f"Report: {_description}", + _type=_type, + published_time=datetime.strptime(_date_published, "%Y-%m-%d"), + related_objects_ids=report_related_objects_ids, + tlp_color=self.tlp_color, + labels=[self.collection, _label, ta_label], + ) + report.set_description(f"Report {_id}: {_description}") + report.generate_external_references(report_links) + report.generate_stix_objects() + + return report + + def generate_stix_yara( + self, obj, json_date_obj=None, related_objects=None, is_ioc=True + ): + if not obj: + return None + + _yara = obj.get("yara") + # _yara_rule_name = obj.get("yara-rule-name") + _context = obj.get("context") + _type = "yara" + _label = "yara" + _date_created = None + + if json_date_obj: + try: + _date_created = datetime.strptime( + json_date_obj.get("date-created"), "%Y-%m-%dT%H:%M:%S%z" + ) + except (Exception,): + self.helper.log_warning( + "Failed to format first_seen: {}. Using default.".format( + json_date_obj.get("date-updated"), + ) + ) + _date_created = datetime.now() + + yara = ds.Indicator( + name=_yara, + _type=_type, + context=_context, + created=_date_created, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + yara.is_ioc = is_ioc + yara.generate_stix_objects() + + if related_objects: + self._generate_relations(yara, related_objects, "indicates") + + yara.add_relationships_to_stix_objects() + + return yara + + def generate_stix_suricata( + self, obj, json_date_obj=None, related_objects=None, is_ioc=True + ): + if not obj: + return None + + _suricata = obj.get("signature") + # _suricata_sid = obj.get("sid") + _context = obj.get("context") + _type = "suricata" + _label = "suricata" + _date_created = None + + if json_date_obj: + try: + _date_created = datetime.strptime( + json_date_obj.get("date-created"), "%Y-%m-%dT%H:%M:%S%z" + ) + except (Exception,): + self.helper.log_warning( + "Failed to format first_seen: {}. Using default.".format( + json_date_obj.get("date-updated"), + ) + ) + _date_created = datetime.now() + + suricata = ds.Indicator( + name=_suricata, + _type=_type, + context=_context, + created=_date_created, + tlp_color=self.tlp_color, + labels=[self.collection, _label], + ) + suricata.is_ioc = is_ioc + suricata.generate_stix_objects() + + if related_objects: + self._generate_relations(suricata, related_objects, "indicates") + + suricata.add_relationships_to_stix_objects() + + return suricata diff --git a/external-import/group-ib/src/config.py b/external-import/group-ib/src/config.py new file mode 100644 index 0000000000..eb79a51ba3 --- /dev/null +++ b/external-import/group-ib/src/config.py @@ -0,0 +1,331 @@ +import os + +from stix2 import TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE +from stix2.v21.vocab import MALWARE_TYPE + + +class Config(object): + + # Set up product metadata + PRODUCT_TYPE = "SCRIPT" + PRODUCT_NAME = "OpenCTI" + PRODUCT_VERSION = "unknown" + INTEGRATION = "GroupIB_TI_OpenCTI_Connector" + INTEGRATION_VERSION = "1.0.0" + + # Author + AUTHOR = "Group-IB" + + # Set project root dir + ROOT_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + + # Set basedirs + DOCS_DIR = os.path.join(ROOT_DIR, "docs") + LOGS_DIR = os.path.join(ROOT_DIR, "log") + + # Set up logging + ROOT_LOGGING_LEVEL = "DEBUG" + LOGGING_FORMAT = ( + "%(asctime)s [%(name)s: %(filename)s.%(lineno)s] [%(levelname)s] %(message)s" + ) + + # Set up logs files + LOGS_SESSION_FILENAME = "session_ti.log" + LOGS_INFO_FILENAME = "info_ti.log" + LOGS_WARNING_FILENAME = "warning_ti.log" + + # Set up config filename + _config_name_yml = "endpoints_config.yaml" + _config_name_json = "mapping.json" + _q_config_name_json = "qmapping.json" + + # Set up configs + CONFIG_YML = os.path.join(DOCS_DIR, "configs", _config_name_yml) + CONFIG_JSON = os.path.join(DOCS_DIR, "configs", _config_name_json) + Q_CONFIG_JSON = os.path.join(DOCS_DIR, "configs", _q_config_name_json) + + # Set up MITRE Matrix + MITRE_CACHE_FILENAME = "mitre_cache.json" + MITRE_CACHE_FOLDER = os.path.join(DOCS_DIR, "cache") + + # Set common mapping variables + STIX_TLP_MAP = { + "white": TLP_WHITE, + "green": TLP_GREEN, + "amber": TLP_AMBER, + "red": TLP_RED, + } + STIX_MAIN_OBSERVABLE_TYPE_MAP = { + "domain": "Domain-Name", + "file": "StixFile", + "ipv4": "IPv4-Addr", + "ipv6": "IPv6-Addr", + "url": "Url", + "yara": "Unknown", + "suricata": "Unknown", + } + STIX_MALWARE_TYPE_MAP = {*MALWARE_TYPE} + # ISO3166-1 https://www.iso.org/standard/72482.html + COUNTRIES = { + "AF": "Afghanistan", + "AX": "Åland Islands", + "AL": "Albania", + "DZ": "Algeria", + "AS": "American Samoa", + "AD": "Andorra", + "AO": "Angola", + "AI": "Anguilla", + "AQ": "Antarctica", + "AG": "Antigua And Barbuda", + "AR": "Argentina", + "AM": "Armenia", + "AW": "Aruba", + "AU": "Australia", + "AT": "Austria", + "AZ": "Azerbaijan", + "BS": "Bahamas", + "BH": "Bahrain", + "BD": "Bangladesh", + "BB": "Barbados", + "BY": "Belarus", + "BE": "Belgium", + "BZ": "Belize", + "BJ": "Benin", + "BM": "Bermuda", + "BT": "Bhutan", + "BO": "Bolivia", + ## "BQ": "Bonaire, Sint Eustatius and Saba", + "BA": "Bosnia and Herzegovina", + "BW": "Botswana", + "BV": "Bouvet Island", + "BR": "Brazil", + "IO": "British Indian Ocean Territory", + "BN": "Brunei Darussalam", + "BG": "Bulgaria", + "BF": "Burkina Faso", + "BI": "Burundi", + "KH": "Cambodia", + "CM": "Cameroon", + "CA": "Canada", + "CV": "Cape Verde", + "KY": "Cayman Islands", + "CF": "Central African Republic", + "TD": "Chad", + "CL": "Chile", + "CN": "China", + "CX": "Christmas Island", + "CC": "Cocos (Keeling) Islands", + "CO": "Colombia", + "KM": "Comoros", + "CG": "Congo", + "CD": "Congo, The Democratic Republic Of The", + "CK": "Cook Islands", + "CR": "Costa Rica", + "CI": "Cote D'ivoire", + "HR": "Croatia", + "CU": "Cuba", + "CW": "Country of Curaçao", + "CY": "Cyprus", + "CZ": "Czech Republic", + "DK": "Denmark", + "DJ": "Djibouti", + "DM": "Dominica", + "DO": "Dominican Republic", + "EC": "Ecuador", + "EG": "Egypt", + "SV": "El Salvador", + "GQ": "Equatorial Guinea", + "ER": "Eritrea", + "EE": "Estonia", + "ET": "Ethiopia", + "FK": "Falkland Islands (Malvinas)", + "FO": "Faroe Islands", + "FJ": "Fiji", + "FI": "Finland", + "FR": "France", + "GF": "French Guiana", + "PF": "French Polynesia", + "TF": "French Southern Territories", + "GA": "Gabon", + "GM": "Gambia", + "GE": "Georgia", + "DE": "Germany", + "GH": "Ghana", + "GI": "Gibraltar", + "GR": "Greece", + "GL": "Greenland", + "GD": "Grenada", + "GP": "Guadeloupe", + "GU": "Guam", + "GT": "Guatemala", + "GG": "Guernsey", + "GN": "Guinea", + "GW": "Guinea-bissau", + "GY": "Guyana", + "HT": "Haiti", + "HM": "Heard Island And Mcdonald Islands", + "VA": "Holy See (Vatican City State)", + "HN": "Honduras", + "HK": "Hong Kong", + "HU": "Hungary", + "IS": "Iceland", + "IN": "India", + "ID": "Indonesia", + "IR": "Iran, Islamic Republic Of", + "IQ": "Iraq", + "IE": "Ireland", + "IM": "Isle Of Man", + "IL": "Israel", + "IT": "Italy", + "JM": "Jamaica", + "JP": "Japan", + "JE": "Jersey", + "JO": "Jordan", + "KZ": "Kazakhstan", + "KE": "Kenya", + "KI": "Kiribati", + "KP": "Korea, Democratic People's Republic Of", + "KR": "Korea, Republic Of", + "KW": "Kuwait", + "KG": "Kyrgyzstan", + "LA": "Lao People's Democratic Republic", + "LV": "Latvia", + "LB": "Lebanon", + "LS": "Lesotho", + "LR": "Liberia", + "LY": "Libyan Arab Jamahiriya", + "LI": "Liechtenstein", + "LT": "Lithuania", + "LU": "Luxembourg", + "MO": "Macao", + "MK": "Macedonia, The Former Yugoslav Republic Of", + "MG": "Madagascar", + "MW": "Malawi", + "MY": "Malaysia", + "MV": "Maldives", + "ML": "Mali", + "MT": "Malta", + "MH": "Marshall Islands", + "MQ": "Martinique", + "MR": "Mauritania", + "MU": "Mauritius", + "YT": "Mayotte", + "MX": "Mexico", + "FM": "Micronesia, Federated States Of", + "MD": "Moldova, Republic Of", + "MC": "Monaco", + "MN": "Mongolia", + "ME": "Montenegro", + "MS": "Montserrat", + "MA": "Morocco", + "MZ": "Mozambique", + "MM": "Myanmar", + "NA": "Namibia", + "NR": "Nauru", + "NP": "Nepal", + "NL": "Netherlands", + "NC": "New Caledonia", + "NZ": "New Zealand", + "NI": "Nicaragua", + "NE": "Niger", + "NG": "Nigeria", + "NU": "Niue", + "NF": "Norfolk Island", + "MP": "Northern Mariana Islands", + "NO": "Norway", + "OM": "Oman", + "PK": "Pakistan", + "PW": "Palau", + "PS": "Palestinian Territory, Occupied", + "PA": "Panama", + "PG": "Papua New Guinea", + "PY": "Paraguay", + "PE": "Peru", + "PH": "Philippines", + "PN": "Pitcairn", + "PL": "Poland", + "PT": "Portugal", + "PR": "Puerto Rico", + "QA": "Qatar", + "RE": "Reunion", + "RO": "Romania", + "RU": "Russian Federation", + "RW": "Rwanda", + ## "BL": "Saint Barthelemy", + "SH": "Saint Helena", + "KN": "Saint Kitts And Nevis", + "LC": "Saint Lucia", + ## "MF": "Saint Martin (French part)", + "PM": "Saint Pierre And Miquelon", + "VC": "Saint Vincent And The Grenadines", + "WS": "Samoa", + "SM": "San Marino", + "ST": "Sao Tome And Principe", + "SA": "Saudi Arabia", + "SN": "Senegal", + "RS": "Serbia", + "SC": "Seychelles", + "SL": "Sierra Leone", + "SG": "Singapore", + ## "SX": "Sint Maarten (Dutch part)", + "SK": "Slovakia", + "SI": "Slovenia", + "SB": "Solomon Islands", + "SO": "Somalia", + "ZA": "South Africa", + "GS": "South Georgia And The South Sandwich Islands", + ## "SS": "South Sudan", + "ES": "Spain", + "LK": "Sri Lanka", + "SD": "Sudan", + "SR": "Suriname", + "SJ": "Svalbard And Jan Mayen", + "SZ": "Swaziland", + "SE": "Sweden", + "CH": "Switzerland", + "SY": "Syrian Arab Republic", + "TW": "Taiwan, Province Of China", + "TJ": "Tajikistan", + "TZ": "Tanzania, United Republic Of", + "TH": "Thailand", + "TL": "Timor-leste", + "TG": "Togo", + "TK": "Tokelau", + "TO": "Tonga", + "TT": "Trinidad And Tobago", + "TN": "Tunisia", + "TR": "Turkey", + "TM": "Turkmenistan", + "TC": "Turks And Caicos Islands", + "TV": "Tuvalu", + "UG": "Uganda", + "UA": "Ukraine", + "AE": "United Arab Emirates", + "GB": "United Kingdom", + "US": "United States", + "UM": "United States Minor Outlying Islands", + "UY": "Uruguay", + "UZ": "Uzbekistan", + "VU": "Vanuatu", + "VE": "Venezuela", + "VN": "Viet Nam", + "VG": "Virgin Islands, British", + "VI": "Virgin Islands, U.S.", + "WF": "Wallis And Futuna", + "EH": "Western Sahara", + "YE": "Yemen", + "ZM": "Zambia", + "ZW": "Zimbabwe", + } + STIX_COUNTRY_TYPE_MAP = { + "country": "Country", + "city": "City", + "state": "Administrative-Area", + } + STIX_REPORT_TYPE_MAP = {"threat_report": "Threat-Report"} + STIX_RELATION_TYPE_MAP = { + "indicator": "based-on", + "attack_pattern": "indicates", + "malware": "indicates", + "threat_actor": "indicates", + } diff --git a/external-import/group-ib/src/data_to_stix2.py b/external-import/group-ib/src/data_to_stix2.py new file mode 100644 index 0000000000..2fc0e4df93 --- /dev/null +++ b/external-import/group-ib/src/data_to_stix2.py @@ -0,0 +1,789 @@ +""" +############################## TERMS OF USE #################################### +# The following code is provided for demonstration purposes only, and should # +# not be used without independent verification. Group-IB makes no # +# representations or warranties, express, implied, statutory, or otherwise, # +# regarding this code, and provides it strictly "as-is". # +# Group-IB shall not be liable for, and you assume all risk of # +# using the foregoing. # +################################################################################ +Author: Pavel Reshetnikov, Integration developer, 2024 +""" + +import ipaddress +import re +from datetime import datetime +from urllib.parse import urlparse + +import pycti # type: ignore +import stix2 + +from config import Config + + +class ConversionError(Exception): + """Generic exception for stix2 conversion issues""" + + pass + + +class _CommonUtils: + + @staticmethod + def _sanitize(message): + # type: (str) -> str + """Sanitize message""" + # Use repr to suppress \t, \n, \r, and strip the surrounding quotes added by repr. + return repr(message)[1:-1] + + @staticmethod + def _remove_html_tags(message): + """Remove html tags from a string""" + clean = re.compile("<.*?>") + return re.sub(clean, "", message) + + @staticmethod + def _extract_domain(url, suffix=""): + # type: (str, str) -> str + """Extract domain name from url""" + parsed_url = urlparse(url) + if parsed_url.path and parsed_url.path != "/": + return parsed_url.netloc + suffix + return parsed_url.netloc + + @staticmethod + def is_ipv4(ipv4): + # type: (str) -> bool + """Determine whether the provided IP string is IPv4.""" + try: + ipaddress.IPv4Address(ipv4) + return True + except ipaddress.AddressValueError: + return False + + @staticmethod + def is_ipv6(ipv6): + # type: (str) -> bool + """Determine whether the provided IP string is IPv6.""" + try: + ipaddress.IPv6Address(ipv6) + return True + except ipaddress.AddressValueError: + return False + + @staticmethod + def determine_hash_algorithm_by_length(file_hash): + """Determine file hash algorithm from length""" + if len(file_hash) == 64: + return "SHA-256" + elif len(file_hash) == 40: + return "SHA-1" + elif len(file_hash) == 32: + return "MD5" + msg = f"Could not determine hash type for {file_hash}. Only MD5, SHA1 and SHA256 hashes are supported" + raise ValueError(msg) + + @staticmethod + def _generate_tlp_obj(color): + # type: (str) -> Any + """Generate TLP object""" + return Config.STIX_TLP_MAP.get(color.lower()) + + @staticmethod + def _generate_main_observable_type(obj_type): + # type: (str) -> str + """Generate TLP object""" + return Config.STIX_MAIN_OBSERVABLE_TYPE_MAP.get(obj_type) + + @staticmethod + def _generate_malware_type(obj_type): + # type: (str) -> Optional[str, None] + """Generate Malware type object""" + if obj_type.lower() in Config.STIX_MALWARE_TYPE_MAP: + return obj_type.lower() + else: + return None + + @staticmethod + def _generate_country_by_cc(country_code): + # type: (str) -> str + """Generate Country by Country Code""" + return Config.COUNTRIES.get(country_code) + + @staticmethod + def _generate_stix_country_type(country_type): + # type: (str) -> str + """Generate STIX2 Country type by Country type""" + return Config.COUNTRIES.get(country_type) + + @staticmethod + def _generate_stix_report_type(report_type): + # type: (str) -> str + """Generate STIX2 Report type by Report type""" + return Config.STIX_REPORT_TYPE_MAP.get(report_type) + + +class BaseEntity(_CommonUtils): + + def __init__(self, name, _type, tlp_color): + self.name = name + self.type = _type + self.author = self._generate_author() + self.tlp = self._generate_tlp_obj(tlp_color) + self.is_ioc = False + self.description = "" + + # defined in self._setup + self.stix_indicator = None + self.stix_observable = None + self.stix_sdo = None + self.stix_common = None + self.stix_relationships = list() + + self.external_references = list() + + self.stix_main_object = None + self.stix_objects = None + + @staticmethod + def _generate_author(): + """Generate Author""" + return stix2.Identity( + id=pycti.Identity.generate_id(Config.AUTHOR, "organization"), + name=Config.AUTHOR, + identity_class="organization", + ) + + def _generate_indicator(self): + return + + def _generate_observable(self): + return + + def _generate_sdo(self): + return + + def _generate_common(self): + return + + def set_description(self, text): + # type: (str) -> None + """Set object description""" + if text: + self.description = self._remove_html_tags(self._sanitize(text)) + + def _generate_relationship(self, source_id, target_id, relation_type="based-on"): + return stix2.Relationship( + id=pycti.StixCoreRelationship.generate_id( + relation_type, source_id, target_id + ), + relationship_type=relation_type, + source_ref=source_id, + target_ref=target_id, + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + ) + + def generate_relationship( + self, source_object, target_object, relation_type="based-on" + ): + # type: (Any, Any, str) -> None + + self.stix_relationships.append( + self._generate_relationship( + source_object.id, target_object.id, relation_type + ) + ) + + def _generate_external_reference(self, ref_id, ref_url, ref_desc): + return stix2.ExternalReference( + # external_id=pycti.ExternalReference.generate_id(ref_url, self._extract_domain(ref_url), ref_id), + source_name=self._extract_domain(ref_url), + url=ref_url, + description=ref_desc, + ) + + def generate_external_references(self, reference_objects): + # type: (List[Tuple[str, str, str]]) -> List[stix2.ExternalReference] + """ + Generate STIX ExternalReference objects from object attributes + + Examples: + [(reference_id, reference_url, reference_description)] + + [ + ( + "349585fa33dd9117622e676d69c4d286fb68b4d3", + "https://tap.group-ib.com/ta/last-threats?threat=349585fa33dd9117622e676d69c4d286fb68b4d3", + "TI Portal external reference" + ) + ] + """ + if reference_objects: + self.external_references = [ + self._generate_external_reference(ref_id, ref_url, ref_desc) + for ref_id, ref_url, ref_desc in reference_objects + ] + else: + self.external_references = [] + return self.external_references + + def generate_stix_objects(self): + """Generate STIX objects from object attributes""" + self.stix_indicator = self._generate_indicator() + self.stix_observable = self._generate_observable() + self.stix_sdo = self._generate_sdo() + self.stix_common = self._generate_common() + if self.is_ioc: + if isinstance(self.stix_indicator, list): + self.stix_objects = [ + _ + for _ in [ + self.stix_observable, + self.stix_sdo, + self.stix_common, + ] + if _ + ] + self.stix_objects += self.stix_indicator + else: + self.stix_objects = [ + _ + for _ in [ + self.stix_indicator, + self.stix_observable, + self.stix_sdo, + self.stix_common, + ] + if _ + ] + return self + else: + self.stix_objects = [ + _ + for _ in [ + self.stix_observable, + self.stix_sdo, + self.stix_common, + ] + if _ + ] + return self + + def add_relationships_to_stix_objects(self): + """Append relationships to STIX objects""" + if self.stix_relationships: + self.stix_objects += self.stix_relationships + return self.stix_objects + + def bundle(self): + """Generate Bundle of STIX objects""" + return stix2.Bundle(objects=self.stix_objects, allow_custom=True) + + +class _BaseIndicator(BaseEntity): + """ + Base class for Indicators of Compromise (IP, Hash, URL, Domain) + + autostart + 1. _generate_indicator + 2. _generate_observable + + 3. + """ + + def __init__(self, name, _type, tlp_color, labels, risk_score): + # type: (str, str, str, List[str], Union[None, str]) -> None + super().__init__(name, _type, tlp_color) + + self.labels = labels + self.risk_score = risk_score + + def _create_pattern(self, pattern_name): + return + + def _generate_indicator(self): + """Creates and returns STIX2 indicator object""" + return stix2.Indicator( + id=pycti.Indicator.generate_id(self.name), + name=self.name, + description=self.description, + pattern_type="stix", + valid_from=datetime.now(), + pattern=self._create_pattern(self.name), + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_main_observable_type": self._generate_main_observable_type( + self.type + ), + "x_opencti_labels": self.labels, + }, + ) + + +class Indicator(_BaseIndicator): + """Converts Indicator to STIX2 indicator""" + + def __init__( + self, + name, + _type, + tlp_color="white", + labels=None, + risk_score=None, + context=None, + created=None, + ): + super().__init__(name, _type, tlp_color, labels, risk_score) + + self.context = context + self.created = created + + def _create_pattern(self, pattern_name): + if pattern_name == "yara": + return self.context + elif pattern_name == "suricata": + return self.context + else: + msg = f"This pattern value {pattern_name} is not a valid." + raise ValueError(msg) + + def _generate_indicator(self): + """Creates and returns STIX2 indicator object""" + self.stix_main_object = stix2.Indicator( + id=pycti.Indicator.generate_id(self.name), + name=self.name, + description=self.description, + pattern=self._create_pattern(self.type), + pattern_type=self.type, + valid_from=datetime.now(), + created=self.created, + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_main_observable_type": self._generate_main_observable_type( + self.type + ), + "x_opencti_labels": self.labels, + }, + ) + return self.stix_main_object + + +class FileHash(_BaseIndicator): + """Converts Hash to STIX2 File indicator and observable""" + + def __init__(self, name, _type, tlp_color="white", labels=None, risk_score=None): + # type: (list, str, str, List[str], Union[None, str]) -> None + super().__init__(name, _type, tlp_color, labels, risk_score) + + def _create_pattern(self, pattern_name): + return f"[file:hashes.'{self.determine_hash_algorithm_by_length(pattern_name)}' = '{pattern_name}']" + + def _generate_observable(self): + self.stix_main_object = stix2.File( + hashes={ + self.determine_hash_algorithm_by_length(_name): _name + for _name in self.name + if _name + }, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + }, + ) + return self.stix_main_object + + def _generate_indicator(self): + """Creates and returns STIX2 indicator object""" + return [ + stix2.Indicator( + id=pycti.Indicator.generate_id(_name), + name=_name, + description=self.description, + pattern_type="stix", + valid_from=datetime.now(), + pattern=self._create_pattern(_name), + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_main_observable_type": self._generate_main_observable_type( + self.type + ), + "x_opencti_labels": self.labels, + }, + ) + for _name in self.name + if _name + ] + + +class IPAddress(_BaseIndicator): + """Converts IP address to STIX2 IP indicator and observable""" + + def __init__(self, name, _type, tlp_color="white", labels=None, risk_score=None): + super().__init__(name, _type, tlp_color, labels, risk_score) + + def _create_pattern(self, pattern_name): + if self.is_ipv4(pattern_name): + return f"[ipv4-addr:value = '{pattern_name}']" + elif self.is_ipv6(pattern_name): + return f"[ipv6-addr:value = '{pattern_name}']" + else: + msg = f"This pattern value {pattern_name} is not a valid IPv4 address." + raise ValueError(msg) + + def _generate_observable(self): + self.stix_main_object = stix2.IPv4Address( + value=self.name, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + }, + ) + return self.stix_main_object + + +class URL(_BaseIndicator): + """Converts URL to STIX2 URL indicator and observable""" + + def __init__(self, name, _type, tlp_color="white", labels=None, risk_score=None): + super().__init__(name, _type, tlp_color, labels, risk_score) + + def _create_pattern(self, pattern_name): + return f"[url:value = '{pattern_name}']" + + def _generate_observable(self): + self.stix_main_object = stix2.URL( + value=self.name, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + }, + ) + return self.stix_main_object + + +class Domain(_BaseIndicator): + """Converts URL to STIX2 URL indicator and observable""" + + def __init__(self, name, _type, tlp_color="white", labels=None, risk_score=None): + super().__init__(name, _type, tlp_color, labels, risk_score) + + def _create_pattern(self, pattern_name): + return f"[domain-name:value = '{pattern_name}']" + + def _generate_observable(self): + self.stix_main_object = stix2.DomainName( + value=self.name, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + }, + ) + return self.stix_main_object + + +class _BaseSDO(BaseEntity): + def __init__(self, name, _type, tlp_color, labels, risk_score): + # type: (str, str, str, List[str], Union[None, str]) -> None + super().__init__(name, _type, tlp_color) + + self.labels = labels + self.risk_score = risk_score + + +class ThreatActor(_BaseSDO): + """Converts Threat Actor to STIX2 Threat Actor SDO""" + + def __init__( + self, + name, + _type, + global_label, + tlp_color="white", + labels=None, + risk_score=None, + aliases=None, + first_seen=None, + last_seen=None, + goals=None, + roles=None, + ): + super().__init__(name, _type, tlp_color, labels, risk_score) + + self.global_label = global_label + self.aliases = aliases + self.first_seen = first_seen + self.last_seen = last_seen + self.goals = goals + self.roles = roles + + def _generate_sdo(self): + self.stix_main_object = stix2.ThreatActor( + id=pycti.ThreatActor.generate_id(self.name), + name=self.name, + aliases=self.aliases, + first_seen=self.first_seen, + last_seen=self.last_seen, + goals=self.goals, + roles=self.roles, + created_by_ref=self.author.id, + threat_actor_types=[self.global_label], + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + }, + ) + return self.stix_main_object + + +class Malware(_BaseSDO): + """Converts Malware to STIX2 Malware SDO""" + + def __init__( + self, + name, + _type, + malware_types, + tlp_color="white", + labels=None, + risk_score=None, + aliases=None, + last_seen=None, + ): + super().__init__(name, _type, tlp_color, labels, risk_score) + + self.malware_types = [] + if malware_types: + self.malware_types = [ + self._generate_malware_type(_t) for _t in malware_types + ] + self.aliases = aliases + self.last_seen = last_seen + + def _generate_sdo(self): + self.stix_main_object = stix2.Malware( + id=pycti.Malware.generate_id(self.name), + name=self.name, + aliases=self.aliases, + last_seen=self.last_seen, + malware_types=self.malware_types or ["unknown"], + is_family=False, + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + }, + ) + return self.stix_main_object + + +class Vulnerability(_BaseSDO): + """Converts Vulnerability to STIX2 Vulnerability SDO""" + + def __init__( + self, + name, + _type, + tlp_color="white", + labels=None, + risk_score=None, + created=None, + modified=None, + cvss_score=None, + cvss_vector=None, + ): + super().__init__(name, _type, tlp_color, labels, risk_score) + + self.created = created + self.cvss_score = cvss_score + self.cvss_vector = cvss_vector + """ + CVSSv2: + "0-3.9": "LOW" + "4.0-6.9": "MEDIUM" + "7.0-10.0": "HIGH" + CVSSv3: + "0.1-3.9": "LOW" + "4.0-6.9": "MEDIUM" + "7.0-8.9": "HIGH" + "9.0-10.0": "CRITICAL" + """ + if self.cvss_score: + if 0 <= self.cvss_score <= 3.9: + self.cvss_severity = "LOW" + elif 4.0 <= self.cvss_score <= 6.9: + self.cvss_severity = "MEDIUM" + elif 7.0 < self.cvss_score <= 8.9: + self.cvss_severity = "HIGH" + elif 9.0 < self.cvss_score <= 10.0: + self.cvss_severity = "CRITICAL" + else: + self.cvss_severity = None + else: + self.cvss_severity = None + + def _generate_sdo(self): + self.stix_main_object = stix2.Vulnerability( + id=pycti.Vulnerability.generate_id(self.name), + name=self.name, + created=self.created, + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + "x_opencti_cvss_base_score": self.cvss_score, + "x_opencti_cvss_base_severity": self.cvss_severity, + "x_opencti_cvss_attack_vector": self.cvss_vector, + }, + ) + return self.stix_main_object + + +class AttackPattern(_BaseSDO): + """Converts AttackPattern to STIX2 AttackPattern SDO""" + + def __init__( + self, + name, + _type, + kill_chain_phases, + mitre_id, + tlp_color="white", + labels=None, + risk_score=None, + ): + super().__init__(name, _type, tlp_color, labels, risk_score) + + self.kill_chain_phases = kill_chain_phases + self.mitre_id = mitre_id + + def _generate_sdo(self): + self.stix_main_object = stix2.AttackPattern( + id=pycti.AttackPattern.generate_id(self.name, self.mitre_id), + name=self.name, + kill_chain_phases=self.kill_chain_phases, + description=self.description, + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + "x_mitre_id": self.mitre_id, + }, + ) + return self.stix_main_object + + +class Report(_BaseSDO): + """Converts AttackPattern to STIX2 AttackPattern SDO""" + + def __init__( + self, + name, + _type, + published_time, + related_objects_ids, + tlp_color="white", + labels=None, + risk_score=None, + ): + super().__init__(name, _type, tlp_color, labels, risk_score) + + self.published_time = published_time or datetime.now() + self.related_objects_ids = related_objects_ids + + def _generate_sdo(self): + self.stix_main_object = stix2.Report( + id=pycti.Report.generate_id(self.name, self.published_time), + name=self.name, + description=self.description, + published=self.published_time, + report_types=[self._generate_stix_report_type(self.type)], + object_refs=self.related_objects_ids, + created_by_ref=self.author.id, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_score": self.risk_score or None, + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + }, + ) + return self.stix_main_object + + +class _BaseCommon(BaseEntity): + def __init__(self, name, _type, tlp_color, labels, risk_score): + # type: (str, str, str, List[str], Union[None, str]) -> None + super().__init__(name, _type, tlp_color) + + self.labels = labels + self.risk_score = risk_score + + +class Location(_BaseCommon): + """Converts Location to STIX2 Location SDO""" + + def __init__( + self, + name, + _type, + tlp_color="white", + labels=None, + risk_score=None, + location_type="Country", + ): + super().__init__(name, _type, tlp_color, labels, risk_score) + + self.location_type = location_type + + def _generate_common(self): + self.stix_main_object = stix2.Location( + id=pycti.Location.generate_id( + self._generate_country_by_cc(self.name), self.location_type + ), + name=self._generate_country_by_cc(self.name), + country=self.name, + object_marking_refs=[self.tlp], + custom_properties={ + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + "x_opencti_aliases": self.name, + }, + ) + return self.stix_main_object + + +class KillChainPhase(_BaseCommon): + """Converts KillChainPhase to STIX2 KillChainPhase SDO""" + + def __init__(self, name, _type, tlp_color="white", labels=None, risk_score=None): + super().__init__(name, _type, tlp_color, labels, risk_score) + + def _generate_common(self): + self.stix_main_object = stix2.KillChainPhase( + kill_chain_name=self.name, + phase_name=self.type, + custom_properties={ + "x_opencti_labels": self.labels, + "x_opencti_external_references": self.external_references, + }, + ) + return self.stix_main_object diff --git a/external-import/group-ib/src/lib/cyberintegrations-0.6.6-py3-none-any.whl b/external-import/group-ib/src/lib/cyberintegrations-0.6.6-py3-none-any.whl new file mode 100644 index 0000000000..b82e9c808e Binary files /dev/null and b/external-import/group-ib/src/lib/cyberintegrations-0.6.6-py3-none-any.whl differ diff --git a/external-import/group-ib/src/lib/external_import.py b/external-import/group-ib/src/lib/external_import.py new file mode 100644 index 0000000000..212ba421b9 --- /dev/null +++ b/external-import/group-ib/src/lib/external_import.py @@ -0,0 +1,321 @@ +import os +import sys +import time +from datetime import datetime + +import stix2 +from cyberintegrations import TIAdapter +from cyberintegrations.decorators import cache_data +from cyberintegrations.utils import FileHandler, ProxyConfigurator +from pycti import OpenCTIConnectorHelper + +from config import Config + + +@cache_data( + cache_dir=Config.MITRE_CACHE_FOLDER, cache_file=Config.MITRE_CACHE_FILENAME, ttl=1 +) +def get_mitre_mapper(adapter, endpoint, params, decode=True, **kwargs): + # type: (TIAdapter, str, dict, bool, dict) -> dict + mitre_mapper = {} + + response = adapter.send_request( + endpoint=endpoint, params=params, decode=decode, **kwargs + ) + + for pattern_dictionary in response.get("AttackPattern").values(): + name = pattern_dictionary.get("name", "") + if name[0] == "[": + name = name[1:-1:] + name = name.split("->")[-1] + mitre_mapper[pattern_dictionary.get("mitreId")] = name + + return mitre_mapper + + +class ExternalImportConnector: + """Specific external-import connector + + This class encapsulates the main actions, expected to be run by + any external-import connector. Note that the attributes defined below + will be complemented per each connector type. + + Attributes: + helper (OpenCTIConnectorHelper): The helper to use. + interval (str): The interval to use. It SHOULD be a string in the format '7d', '12h', '10m', '30s' + where the final letter SHOULD be one of 'd', 'h', 'm', 's' standing for day, hour, minute, second respectively. + update_existing_data (str): Whether to update existing data or not in OpenCTI. + """ + + def __init__(self): + self.helper = OpenCTIConnectorHelper({}) + + # Specific connector attributes for external import connectors + try: + self.interval = os.environ.get("CONNECTOR_RUN_EVERY", None).lower() + self.helper.log_info( + f"Verifying integrity of the CONNECTOR_RUN_EVERY value: '{self.interval}'" + ) + unit = self.interval[-1] + if unit not in ["d", "h", "m", "s"]: + raise TypeError + int(self.interval[:-1]) + except TypeError as ex: + msg = ( + f"Error ({ex}) when grabbing CONNECTOR_RUN_EVERY environment variable: '{self.interval}'. " + "It SHOULD be a string in the format '7d', '12h', '10m', '30s' where the final letter " + "SHOULD be one of 'd', 'h', 'm', 's' standing for day, hour, minute, second respectively. " + ) + self.helper.log_error(msg) + raise ValueError(msg) from ex + + update_existing_data = os.environ.get("CONNECTOR_UPDATE_EXISTING_DATA", "false") + if isinstance(update_existing_data, str) and update_existing_data.lower() in [ + "true", + "false", + ]: + self.update_existing_data = update_existing_data.lower() == "true" + elif isinstance(update_existing_data, bool) and update_existing_data in [ + True, + False, + ]: + self.update_existing_data = update_existing_data + else: + msg = ( + f"Error when grabbing CONNECTOR_UPDATE_EXISTING_DATA environment variable: '{update_existing_data}'. " + "It SHOULD be either `true` or `false`. `false` is assumed. " + ) + self.helper.log_warning(msg) + self.update_existing_data = "false" + + self.cfg = Config + self.fh = FileHandler() + self.pc = ProxyConfigurator() + + self.endpoints_config = self.fh.read_yaml_config(config=Config.CONFIG_YML) + self.mapping_config = self.fh.read_json_config(config=Config.CONFIG_JSON) + + self.ti_api_url = os.environ.get("TI_API_URL") + self._ti_api_username = os.environ.get("TI_API_USERNAME") + self._ti_api_token = os.environ.get("TI_API_TOKEN") + + self.proxy_ip = os.environ.get("PROXY_IP") + self.proxy_port = os.environ.get("PROXY_PORT") + self.proxy_protocol = os.environ.get("PROXY_PROTOCOL") + self._proxy_username = os.environ.get("PROXY_USERNAME") + self._proxy_password = os.environ.get("PROXY_PASSWORD") + + # Global collections filters + self.IGNORE_NON_MALWARE_DDOS = os.environ.get("IGNORE_NON_MALWARE_DDOS") + self.IGNORE_NON_INDICATOR_THREATS = os.environ.get( + "IGNORE_NON_INDICATOR_THREAT_REPORTS" + ) + + # gather TI API creds + self.creds = {"api_key": self._ti_api_token, "username": self._ti_api_username} + # Proxy initialization + self.proxies = self.pc.get_proxies( + proxy_ip=self.proxy_ip, + proxy_port=self.proxy_port, + proxy_protocol=self.proxy_protocol, + proxy_username=self._proxy_username, + proxy_password=self._proxy_password, + ) + # TI API initialization + self.ti_adapter = TIAdapter( + ti_creds_dict=self.creds, + proxies=self.proxies, + config_obj=Config, + api_url=self.ti_api_url, + ) + # create list of collections feeds generators + self.generators_list = None + self.MITRE_MAPPER = None + + def _collect_intelligence(self, collection, portion, mitre_mapper) -> list: + """Collect intelligence from the source""" + raise NotImplementedError + + def _get_interval(self) -> int: + """Returns the interval to use for the connector + + This SHOULD always return the interval in seconds. If the connector expects + the parameter to be received as hours uncomment as necessary. + """ + unit = self.interval[-1:] + value = self.interval[:-1] + + try: + if unit == "d": + # In days: + return int(value) * 60 * 60 * 24 + if unit == "h": + # In hours: + return int(value) * 60 * 60 + if unit == "m": + # In minutes: + return int(value) * 60 + if unit == "s": + # In seconds: + return int(value) + except Exception as ex: + self.helper.log_error( + f"Error when converting CONNECTOR_RUN_EVERY environment variable: '{self.interval}'. {str(ex)}" + ) + raise ValueError( + f"Error when converting CONNECTOR_RUN_EVERY environment variable: '{self.interval}'. {str(ex)}" + ) from ex + + def run(self) -> None: + # Main procedure + self.helper.log_info(f"Starting {self.helper.connect_name} connector...") + while True: + try: + # Get the current timestamp and check + timestamp = int(time.time()) + current_state = self.helper.get_state() + if current_state is not None and "last_run" in current_state: + last_run = current_state["last_run"] + self.helper.log_info( + f"{self.helper.connect_name} connector last run: " + f'{datetime.utcfromtimestamp(last_run).strftime("%Y-%m-%d %H:%M:%S")}' + ) + else: + last_run = None + self.helper.log_info( + f"{self.helper.connect_name} connector has never run" + ) + + # If the last_run is more than interval-1 day + if last_run is None or ((timestamp - last_run) >= self._get_interval()): + self.helper.metric.inc("run_count") + self.helper.metric.state("running") + self.helper.log_info(f"{self.helper.connect_name} will run!") + now = datetime.utcfromtimestamp(timestamp) + friendly_name = f'{self.helper.connect_name} run @ {now.strftime("%Y-%m-%d %H:%M:%S")}' + work_id = self.helper.api.work.initiate_work( + self.helper.connect_id, friendly_name + ) + + try: + # create list of collections feeds generators + self.generators_list = self.ti_adapter.create_generators( + sleep_amount=1 + ) + + # MITRE + self.MITRE_MAPPER = get_mitre_mapper( + adapter=self.ti_adapter, + endpoint="common/matrix/vocab/techniques", + params={}, + ) + + ### + for collection, generator in self.generators_list: + time.sleep(3) + + if not generator: + self.helper.log_warning( + "No generator for collection: {}".format(collection) + ) + continue + + endpoints_config = self.fh.read_yaml_config( + config=Config.CONFIG_YML + ) + if not endpoints_config["collections"][collection][ + "enable" + ]: + self.helper.log_warning( + "User disable collection: {}. Aborting!".format( + collection + ) + ) + continue + + for portion in generator: + + # Extra processing for collections + if ( + collection == "attacks/ddos" + and self.IGNORE_NON_MALWARE_DDOS + ): + parsed_portion = portion.parse_portion( + filter_map=[("malware", [])], + check_existence=True, + ) + elif ( + collection in ["apt/threat", "hi/threat"] + and self.IGNORE_NON_INDICATOR_THREATS + ): + parsed_portion = portion.parse_portion( + filter_map=[("indicators", [])], + check_existence=True, + ) + else: + parsed_portion = portion.parse_portion() + + size = len(parsed_portion) + count = 0 + for event in parsed_portion: + count += 1 + self.helper.log_debug(f"Parsing {count}/{size}") + + bundle_objects = self._collect_intelligence( + collection, event, self.MITRE_MAPPER + ) + bundle = stix2.Bundle( + objects=bundle_objects, allow_custom=True + ).serialize() + + self.helper.log_info( + f"Sending {len(bundle_objects)} STIX objects to OpenCTI..." + ) + self.helper.send_stix2_bundle( + bundle, + update=self.update_existing_data, + work_id=work_id, + ) + + except Exception as e: + self.helper.log_error(str(e)) + + # Store the current timestamp as a last run + message = f"{self.helper.connect_name} connector successfully run, storing last_run as {timestamp}" + self.helper.log_info(message) + + self.helper.log_debug( + f"Grabbing current state and update it with last_run: {timestamp}" + ) + current_state = self.helper.get_state() + if current_state: + current_state["last_run"] = timestamp + else: + current_state = {"last_run": timestamp} + self.helper.set_state(current_state) + + self.helper.api.work.to_processed(work_id, message) + self.helper.log_info( + f"Last_run stored, next run in: {round(self._get_interval() / 60 / 60, 2)} hours" + ) + else: + self.helper.metric.state("idle") + new_interval = self._get_interval() - (timestamp - last_run) + self.helper.log_info( + f"{self.helper.connect_name} connector will not run, " + f"next run in: {round(new_interval / 60 / 60, 2)} hours" + ) + + except (KeyboardInterrupt, SystemExit): + self.helper.log_info(f"{self.helper.connect_name} connector stopped") + sys.exit(0) + except Exception as e: + self.helper.metric.inc("error_count") + self.helper.metric.state("stopped") + self.helper.log_error(str(e)) + + if self.helper.connect_run_and_terminate: + self.helper.log_info(f"{self.helper.connect_name} connector ended") + sys.exit(0) + + time.sleep(60) diff --git a/external-import/group-ib/src/main.py b/external-import/group-ib/src/main.py new file mode 100644 index 0000000000..cf08aaa7e4 --- /dev/null +++ b/external-import/group-ib/src/main.py @@ -0,0 +1,928 @@ +# import os +import sys +import time + +# WARN: python-dotenv is used for integration manual run +import dotenv + +from adapter import DataToSTIXAdapter +from lib.external_import import ExternalImportConnector + +dotenv.load_dotenv() + + +class CustomConnector(ExternalImportConnector): + def __init__(self): + """Initialization of the connector + + Note that additional attributes for the connector can be set after the super() call. + + Standardized way to grab attributes from environment variables is as follows: + + >>> ... + >>> super().__init__() + >>> self.my_attribute = os.environ.get("MY_ATTRIBUTE", "INFO") + + This will make use of the `os.environ.get` method to grab the environment variable and set a default value (in the example "INFO") if it is not set. + Additional tunning can be made to the connector by adding additional environment variables. + + Raising ValueErrors or similar might be useful for tracking down issues with the connector initialization. + """ + super().__init__() + + def _collect_intelligence(self, collection, event, mitre_mapper) -> []: + """Collects intelligence from channels + + Add your code depending on the use case as stated at https://docs.opencti.io/latest/development/connectors/. + Some sample code is provided as a guide to add a specific observable and a reference to the main object. + Consider adding additional methods to the class to make the code more readable. + + Returns: + stix_objects: A list of STIX2 objects.""" + self.helper.log_debug( + f"{self.helper.connect_name} connector is starting the collection of objects..." + ) + + # =========================== + # === Add your code below === + # =========================== + + self.helper.log_debug("Collecting data") + + stix_objects = list() + + # ++ + if collection in ["apt/threat", "hi/threat"]: + + json_threat_report_obj = event.get("threat_report", {}) + json_file_obj = event.get("file", {}) + json_network_obj = event.get("network", {}) + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug(json_threat_actor_obj.get("name")) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + ) + ) + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[stix_threat_actor], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=True, + ) + ) + stix_file_list = report_adapter.generate_stix_file( + obj=json_file_obj, related_objects=[stix_threat_actor], is_ioc=True + ) + + # report_related_objects_ids = [ + # # Files + # *[sf.stix_main_object.id for sf in stix_file_list], + # *[ind.id for sf in stix_file_list for ind in sf.stix_indicator], + # *[rel.id for rel_list in [sf.stix_relationships for sf in stix_file_list] for rel in rel_list], + # # Domains + # *[sf.stix_main_object.id for sf in stix_domain_list], + # *[rel.id for rel_list in [sf.stix_relationships for sf in stix_domain_list] for rel in rel_list], + # # URLs + # *[sf.stix_main_object.id for sf in stix_url_list], + # *[rel.id for rel_list in [sf.stix_relationships for sf in stix_url_list] for rel in rel_list], + # # IPs + # *[sf.stix_main_object.id for sf in stix_ip_list], + # *[sf.stix_indicator.id for sf in stix_ip_list], + # *[rel.id for rel_list in [sf.stix_relationships for sf in stix_ip_list] for rel in rel_list], + # # Malware + # *[sf.stix_main_object.id for sf in stix_malware_list], + # *[rel.id for rel_list in [sf.stix_relationships for sf in stix_malware_list] for rel in rel_list], + # # Vulnerability + # *[sf.stix_main_object.id for sf in stix_vulnerability_list], + # *[rel.id for rel_list in [sf.stix_relationships for sf in stix_vulnerability_list] for rel in rel_list], + # # MITRE + # *[sf.stix_main_object.id for sf in stix_attack_pattern_list], + # *[rel.id for rel_list in [sf.stix_relationships for sf in stix_attack_pattern_list] for rel in rel_list], + # # Threat Actor + # stix_threat_actor.stix_main_object.id, + # *[rel.id for rel in stix_threat_actor.stix_relationships], + # # Locations + # * [sf.stix_main_object.id for sf in stix_threat_actor_location_list] + # ] + + x = list() + if stix_file_list: + [x.extend(ob.stix_objects) for ob in stix_file_list] + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + if stix_attack_pattern_list: + [x.extend(ob.stix_objects) for ob in stix_attack_pattern_list] + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + # TODO: + # - dates parser should be added to objects (report) ++ + # - add types/labels to mapping.json: ++ + # - "data_label": "threat_actor" ++ + # - "data_type": "nation-state" ++ + # - add "targets" countries for TA along with "located-at" ++ + # - add URL, Domain stix objects ++ + # - remove URL as IP or domains as IP ++ + # - check COUNTRIES -- + # - extend suspicious_ip collections with attributed data -- + # - add phishing, deface collections -- + # - finalize README.md ++ + # - bug with relationship "uses" for TA vulnerability -- + # - add valid_from, valid_until to Indicator (file, network) -- + # - add apply_hunting_rules=1 and tailored tag -- + # - hash error ++ + # 388996fdb916fcOef12677531d8f2e0a + # "Invalid value for Indicator 'pattern': FAIL: '388996fdb916fcOef12677531d8f2e0a' is not a valid MD5 hash", "exc_info": "Traceback (most recent call last):\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/src/lib/external_import.py\", line 169, in run\n bundle_objects = self._collect_intelligence(collection, parsed_portion, MITRE_MAPPER)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/src/main.py\", line 110, in _collect_intelligence\n stix_file_list = report_adapter.generate_stix_file(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/src/adapter.py\", line 288, in generate_stix_file\n file.generate_stix_objects()\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/src/data_to_stix2.py\", line 225, in generate_stix_objects\n self.stix_indicator = self._generate_indicator()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/src/data_to_stix2.py\", line 333, in _generate_indicator\n return [\n ^\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/src/data_to_stix2.py\", line 334, in \n stix2.Indicator(\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/venv/lib/python3.11/site-packages/stix2/v21/sdo.py\", line 250, in __init__\n super(Indicator, self).__init__(*args, **kwargs)\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/venv/lib/python3.11/site-packages/stix2/base.py\", line 232, in __init__\n self._check_object_constraints()\n File \"/home/hack/PycharmProjects/Integrations/OpenCTI/connectors/external-import/group-ib/venv/lib/python3.11/site-packages/stix2/v21/sdo.py\", line 270, in _check_object_constraints\n raise InvalidValueError(self.__class__, 'pattern', str(errors[0]))\nstix2.exceptions.InvalidValueError: Invalid value for Indicator 'pattern': FAIL: 'p' is not a valid MD5 hash"} + + stix_report = report_adapter.generate_stix_report( + obj=json_threat_report_obj, + json_date_obj=json_date_obj, + report_related_objects_ids=[_.id for _ in x], + json_malware_report_obj=json_malware_report_obj, + json_threat_actor_obj=json_threat_actor_obj, + ) + + self.helper.log_debug("Pack objects") + + if stix_report: + x += stix_report.stix_objects + x += [stix_report.author] + x += [stix_report.tlp] + + stix_objects += x + + # ++ + if collection in ["apt/threat_actor", "hi/threat_actor"]: + + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + json_date_obj=json_date_obj, + ) + ) + + x = list() + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + stix_objects += x + + # ++ + if collection in ["attacks/ddos"]: + + json_network_obj = event.get("network", {}) + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + ) + ) + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=False, + ) + ) + + x = list() + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + if stix_attack_pattern_list: + [x.extend(ob.stix_objects) for ob in stix_attack_pattern_list] + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + stix_objects += x + + # ++ + if collection in ["attacks/deface"]: + + json_network_obj = event.get("network", {}) + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + ) + ) + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=False, + ) + ) + + x = list() + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + if stix_attack_pattern_list: + [x.extend(ob.stix_objects) for ob in stix_attack_pattern_list] + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + stix_objects += x + + # ++ + if collection in ["attacks/phishing_group"]: + + json_network_obj = event.get("network", {}) + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + ) + ) + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=False, + ) + ) + + x = list() + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + if stix_attack_pattern_list: + [x.extend(ob.stix_objects) for ob in stix_attack_pattern_list] + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + stix_objects += x + + # ++ + if collection in ["attacks/phishing_kit"]: + + json_file_obj = event.get("file", {}) + json_network_obj = event.get("network", {}) + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + ) + ) + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=False, + ) + ) + stix_file_list = report_adapter.generate_stix_file( + obj=json_file_obj, related_objects=[stix_threat_actor], is_ioc=False + ) + + x = list() + if stix_file_list: + [x.extend(ob.stix_objects) for ob in stix_file_list] + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + if stix_attack_pattern_list: + [x.extend(ob.stix_objects) for ob in stix_attack_pattern_list] + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + stix_objects += x + + # ++ + if collection in ["malware/malware"]: + + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj, json_date_obj=json_date_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + json_date_obj=json_date_obj, + ) + ) + + x = list() + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + stix_objects += x + + # ++ + if collection in ["malware/signature"]: + + json_yara_obj = event.get("yara_report", {}) + json_suricata_obj = event.get("suricata_report", {}) + json_network_obj = event.get("network", {}) + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + ) + ) + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=False, + ) + ) + stix_yara = report_adapter.generate_stix_yara( + obj=json_yara_obj, json_date_obj=json_date_obj, is_ioc=True + ) + stix_suricata = report_adapter.generate_stix_suricata( + obj=json_suricata_obj, json_date_obj=json_date_obj, is_ioc=True + ) + + x = list() + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + if stix_attack_pattern_list: + [x.extend(ob.stix_objects) for ob in stix_attack_pattern_list] + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + if stix_yara: + x += stix_yara.stix_objects + if stix_suricata: + x += stix_suricata.stix_objects + + stix_objects += x + + # ++ + if collection in ["malware/yara"]: + + json_yara_obj = event.get("yara_report", {}) + json_suricata_obj = event.get("suricata_report", {}) + json_network_obj = event.get("network", {}) + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + ) + ) + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=False, + ) + ) + stix_yara = report_adapter.generate_stix_yara( + obj=json_yara_obj, + related_objects=[stix_malware_list], + json_date_obj=json_date_obj, + is_ioc=True, + ) + stix_suricata = report_adapter.generate_stix_suricata( + obj=json_suricata_obj, + related_objects=[stix_malware_list], + json_date_obj=json_date_obj, + is_ioc=True, + ) + + x = list() + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + if stix_attack_pattern_list: + [x.extend(ob.stix_objects) for ob in stix_attack_pattern_list] + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + if stix_yara: + x += stix_yara.stix_objects + if stix_suricata: + x += stix_suricata.stix_objects + + stix_objects += x + + # ++ + if collection in ["osi/vulnerability"]: + + json_malware_report_obj = event.get("malware_report", {}) + json_threat_actor_obj = event.get("threat_actor", {}) + json_vulnerability_obj = event.get("vulnerability", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_mitre_matrix_obj = event.get("mitre_matrix", {}) + json_cvss_obj = event.get("cvssv3", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_malware_list = report_adapter.generate_stix_malware( + obj=json_malware_report_obj, json_date_obj=json_date_obj + ) + stix_attack_pattern_list = report_adapter.generate_stix_attack_pattern( + obj=json_mitre_matrix_obj + ) + stix_vulnerability_list = report_adapter.generate_stix_vulnerability( + obj=json_vulnerability_obj, + related_objects=[ + # stix_threat_actor + ], + json_date_obj=json_date_obj, + json_cvss_obj=json_cvss_obj, + ) + stix_threat_actor, stix_threat_actor_location_list = ( + report_adapter.generate_stix_threat_actor( + obj=json_threat_actor_obj, + related_objects=[ + stix_attack_pattern_list, + stix_malware_list, + stix_vulnerability_list, + ], + json_date_obj=json_date_obj, + ) + ) + + x = list() + if stix_malware_list: + [x.extend(ob.stix_objects) for ob in stix_malware_list] + if stix_vulnerability_list: + [x.extend(ob.stix_objects) for ob in stix_vulnerability_list] + if stix_threat_actor: + x += stix_threat_actor.stix_objects + if stix_threat_actor_location_list: + [x.extend(ob.stix_objects) for ob in stix_threat_actor_location_list] + + stix_objects += x + + # ++ + if collection in [ + "suspicious_ip/open_proxy", + "suspicious_ip/scanner", + "suspicious_ip/socks_proxy", + "suspicious_ip/tor_node", + "suspicious_ip/vpn", + ]: + + json_network_obj = event.get("network", {}) + json_evaluation_obj = event.get("evaluation", {}) + json_date_obj = event.get("date", {}) + + self.helper.log_debug("Initializing adapter") + + report_adapter = DataToSTIXAdapter( + mitre_mapper=mitre_mapper, + collection=collection, + tlp_color=json_evaluation_obj.get("tlp", "white"), + helper=self.helper, + is_ioc=True, + ) + + self.helper.log_debug("Generating STIX objects") + + stix_domain_list, stix_url_list, stix_ip_list = ( + report_adapter.generate_stix_network( + obj=json_network_obj, + related_objects=[], + domain_is_ioc=False, + url_is_ioc=False, + ip_is_ioc=False, + ) + ) + + x = list() + if stix_domain_list: + [x.extend(ob.stix_objects) for ob in stix_domain_list] + if stix_url_list: + [x.extend(ob.stix_objects) for ob in stix_url_list] + if stix_ip_list: + [x.extend(ob.stix_objects) for ob in stix_ip_list] + + stix_objects += x + + # =========================== + # === Add your code above === + # =========================== + + self.helper.log_info( + f"{len(stix_objects)} STIX2 objects have been compiled by {self.helper.connect_name} connector. " + ) + return stix_objects + + +if __name__ == "__main__": + try: + connector = CustomConnector() + connector.run() + except Exception as e: + print(e) + time.sleep(10) + sys.exit(0)