[Marketplace Contribution] Github Maltrail Feed

Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py

@@ -0,0 +1,209 @@
+import demistomock as demisto  # noqa: F401
+from CommonServerPython import *  # noqa: F401
+import requests
+import base64
+from datetime import datetime
+import regex
+
+# CONSTANTS
+SOURCE_NAME = "Github Maltrail Feed"
+DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+COMMIT_LIMIT = 100
+
+# ############################## OVERWRITE REGEX FORMATTING ###############################
+regexFlags = re.M  # Multi line matching
+REGEX_IP = r"\b(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(?:\[\.\]|\.)){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\b"
+
+class Client(BaseClient):
+
+    def __init__(self, params: dict):
+        self._verify: bool = not params.get('insecure', False)
+        self.user = params.get('user')
+        self.token = (params.get('api_token') or {}).get('password', '')
+        self.repo = params.get('repository')
+        self.url = params.get('base_url')
+        # self.base_url = f'{self.url}/{self.user}/{self.repo}'
+        self.base_url = urljoin(self.url, self.user)
+        self.base_url = urljoin(self.base_url, self.repo)
+        handle_proxy()
+
+    def http_request_indicators(self):
+        res = requests.get(
+            url=self.base_url,
+            verify=self._verify
+        )
+        try:
+            res.raise_for_status()
+        except Exception:
+            demisto.info(f'Github Maltrail Feed - exception in request: {res.status_code!r} {res.content!r}')
+            raise
+        return res.text
+
+    def getclienturl(self):
+        return self.base_url
+
+    def http_request(self, url_endpoint, params: dict = None):
+        """The HTTP request for daily feeds.
+        Returns:
+            list. A list of indicators fetched from the feed.
+        """
+        self.headers = {
+            'Authorization': "Bearer " + self.token
+        }
+        res = requests.request(
+            method="GET",
+            url=urljoin(self.base_url, url_endpoint),
+            verify=self._verify,
+            headers=self.headers,
+            params=params
+        )
+        return res
+
+
+def fetch_indicators(client: Client, url: str, limit: int=None, params: dict=None):
+    if params:
+        feed_tags = argToList(params.get('feedTags', []))
+        tlp_color = params.get('tlp_color')
+    response = client.http_request(url)
+    indicators_list = []
+    demisto.debug('Fetch of indicators started ###')
+
+    if response.ok:
+        content = response.json()["content"]
+        file_content = base64.b64decode(content).decode("utf-8")
+        lines = file_content.split("\n")
+        for line in lines:
+            if '#' not in line and line != '':
+                type_ = auto_detect_indicator_type(line)
+                if regex.search(REGEX_IP, line):
+                    if line.startswith('http://'):
+                        line = line.removeprefix('http://')
+                    elif line.startswith('https://'):
+                        line = line.removeprefix('https://')
+                    else:
+                        line = line.split(':')[0]
+                    type_ = "IP"
+                elif type_ == "URL":
+                    if not line.startswith('http://') and not line.startswith('https://'):
+                        line = 'http://' + line
+                raw_data = {
+                    'value': line,
+                    'type': type_,
+                }
+                indicator_obj = {
+                    'value': line,
+                    'type': type_,
+                    'service': "GitHub Maltrail Feed",
+                    'fields': {},
+                    'rawJSON': raw_data
+                }
+                if feed_tags:
+                    indicator_obj['fields']['tags'] = feed_tags
+                if tlp_color:
+                    indicator_obj['fields']['trafficlightprotocol'] = tlp_color
+                indicators_list.append(indicator_obj)
+                # If limit is reached, break loop
+                if limit and isinstance(limit, int):
+                    if len(indicators_list)>=limit:
+                        break
+    else:
+        demisto.error(f"Error: {response.status_code} - {response.json()['message']}")
+    return indicators_list
+
+
+def get_last_commit_date(client):
+    api_url = "/commits"
+    response = client.http_request(api_url)
+    last_commit_date = None
+    if response.ok:
+        commits = []
+        page = 1
+        while response.ok and page < COMMIT_LIMIT:
+            commits.extend(response.json())
+            link_header = response.headers.get('Link')
+            if not link_header or 'rel="next"' not in link_header:
+                break
+            page += 1
+            response = client.http_request(api_url, params={'page': page})
+        for commit in commits:
+            if 'qakbot' in commit['commit']['message']:
+                commit_date = date_to_timestamp(parse_date_string(commit['commit']['author']['date'], DATE_FORMAT))
+                if not last_commit_date:
+                    last_commit_date = commit_date
+                elif commit_date > last_commit_date:
+                    last_commit_date = commit_date
+
+    return last_commit_date
+
+
+def fetch_indicators_command(client: Client, params: dict=None):
+    integration_context = get_integration_context()
+    api_url = "/contents/trails/static/malware/qakbot.txt"
+    indicators_list = []
+    #First Fetch
+    if not integration_context:
+        time_of_first_fetch = date_to_timestamp(datetime.now(), DATE_FORMAT)
+        set_integration_context({'time_of_last_fetch': time_of_first_fetch})
+        indicators_list = fetch_indicators(client, api_url, None, params)
+    else:
+        time_from_last_update = integration_context.get('time_of_last_fetch')
+        now = date_to_timestamp(datetime.now(), DATE_FORMAT)
+        last_commit_date = get_last_commit_date(client)
+        if last_commit_date > time_from_last_update:
+            indicators_list = fetch_indicators(client, api_url, None, params)
+            set_integration_context({'time_of_last_fetch': now})
+        else:
+            demisto.debug(f'### Nothing to fetch')
+
+    return indicators_list
+
+
+def get_indicators_command(client: Client, params: dict, args: dict):
+    try:
+        limit = int(args.get('limit', 50))
+    except ValueError:
+        raise ValueError('The limit argument must be a number.')
+    api_url = "/contents/trails/static/malware/qakbot.txt"
+    indicators_list = fetch_indicators(client, api_url, limit, params)
+    entry_result = indicators_list[:limit]
+    human_readable = tableToMarkdown("Indicators from Github Maltrail:", entry_result,
+                                     headers=['value', 'type', 'firstseenbysource', 'lastseenbysource', 'name'],
+                                     removeNull=True)
+    return human_readable, {}, entry_result
+
+
+def test_module_command(client: Client, params: dict, args: dict):
+    client.http_request_indicators()
+    return 'ok', {}, {}
+
+
+def main():
+    params = demisto.params()
+    args = demisto.args()
+
+    command = demisto.command()
+    demisto.info(f'Command being called is {command}')
+
+    # Switch case
+    commands = {
+        'test-module': test_module_command,
+        'gh-maltrail-get-indicators': get_indicators_command
+    }
+
+    try:
+        client = Client(params)
+        if command == 'fetch-indicators':
+            indicators = fetch_indicators_command(client, params)
+            for b in batch(indicators, batch_size=2000):
+                demisto.createIndicators(b)
+        else:
+            readable_output, outputs, raw_response = commands[command](client, params, args)
+            return_outputs(readable_output, outputs, raw_response)
+
+    except Exception as e:
+        raise Exception(f'Error in {SOURCE_NAME} Integration [{e}]')
+
+
+
+if __name__ in ('__main__', '__builtin__', 'builtins'):
+    main()

Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.yml

@@ -0,0 +1,123 @@
+category: Utilities
+commonfields:
+  id: Github Maltrail Feed
+  version: -1
+configuration:
+- additionalinfo: API Token
+  display: ""
+  displaypassword: API Token
+  hiddenusername: true
+  name: api_token
+  required: true
+  type: 9
+- display: 'Username of the repository owner, for example: github.com/repos/{user}/{repo}/issues'
+  name: user
+  required: true
+  type: 0
+- defaultvalue: https://api.github.com/repos
+  display: Base URL
+  name: base_url
+  required: true
+  type: 0
+- display: 'The name of the requested repository, for example: github.com/repos/{user}/{repo}/issues'
+  name: repository
+  required: true
+  type: 0
+- advanced: true
+  display: Trust any certificate (not secure)
+  name: insecure
+  required: false
+  section: Connect
+  type: 8
+- advanced: true
+  display: Use system proxy settings
+  name: proxy
+  required: false
+  section: Connect
+  type: 8
+- defaultvalue: "15"
+  display: Feed Fetch Interval
+  name: feedFetchInterval
+  required: false
+  type: 19
+- defaultvalue: "true"
+  display: Fetch indicators
+  name: feed
+  required: false
+  type: 8
+- additionalinfo: Indicators from this integration instance will be marked with this reputation
+  defaultvalue: Bad
+  display: Indicator Reputation
+  name: feedReputation
+  options:
+  - None
+  - Good
+  - Suspicious
+  - Bad
+  required: false
+  type: 18
+- additionalinfo: Reliability of the source providing the intelligence data
+  defaultvalue: F - Reliability cannot be judged
+  display: Source Reliability
+  name: feedReliability
+  options:
+  - A - Completely reliable
+  - B - Usually reliable
+  - C - Fairly reliable
+  - D - Not usually reliable
+  - E - Unreliable
+  - F - Reliability cannot be judged
+  required: true
+  type: 15
+- defaultvalue: indicatorType
+  display: ""
+  name: feedExpirationPolicy
+  options:
+  - never
+  - interval
+  - indicatorType
+  - suddenDeath
+  required: false
+  type: 17
+- defaultvalue: "20160"
+  display: ""
+  name: feedExpirationInterval
+  required: false
+  type: 1
+- additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system.
+  display: Bypass exclusion list
+  name: feedBypassExclusionList
+  required: false
+  type: 8
+- additionalinfo: Supports CSV values.
+  display: Tags
+  name: feedTags
+  required: false
+  type: 0
+- additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed
+  display: Traffic Light Protocol Color
+  name: tlp_color
+  options:
+  - RED
+  - AMBER
+  - GREEN
+  - WHITE
+  required: false
+  type: 15
+description: Fetches Indicators from Github Repo https://github.com/stamparm/maltrail
+display: Github Maltrail Feed
+name: Github Maltrail Feed
+script:
+  commands:
+  - arguments:
+    - name: limit
+      description: The maximum number of results to return to the output.
+      defaultValue: "50"
+    name: gh-maltrail-get-indicators
+    description: Get indicators from the feed. 
+  dockerimage: demisto/python3:3.10.13.78623
+  feed: true
+  runonce: false
+  script: ''
+  subtype: python3
+  type: python

Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed_description.md

@@ -0,0 +1 @@
+Fetches Indicators from Github Repo https://github.com/stamparm/maltrail

Packs/GithubMaltrailFeed/pack_metadata.json

@@ -0,0 +1,21 @@
+{
+    "name": "Github Maltrail Feed",
+    "description": "Maltrail is a malicious traffic detection system, utilizing publicly available (black)lists containing malicious and/or generally suspicious trails, along with static trails compiled from various AV reports and custom user defined lists, where trail can be anything from domain name (e.g. zvpprsensinaix.com for Banjori malware), URL (e.g. hXXp://109.162.38.120/harsh02.exe for known malicious executable), IP address (e.g. 185.130.5.231 for known attacker) or HTTP User-Agent header value (e.g. sqlmap for automatic SQL injection and database takeover tool). Also, it uses (optional) advanced heuristic mechanisms that can help in discovery of unknown threats (e.g. new malware).\n\nhttps://github.com/stamparm/maltrail",
+    "support": "community",
+    "currentVersion": "1.0.0",
+    "author": "Abel S. Santamarina",
+    "url": "",
+    "email": "",
+    "created": "2023-10-04T21:52:43Z",
+    "categories": ['Utilities', 'Forensics & Malware Analysis', 'Data Enrichment & Threat Intelligence'],
+    "tags": [],
+    "useCases": [],
+    "keywords": [],
+    "marketplaces": [
+        "xsoar",
+        "marketplacev2"
+    ],
+    "githubUser": [
+        "asantamarina"
+    ]
+}