Merge pull request #2092 from blacklanternsecurity/delete-secretsdb

Replace secretsdb with trufflehog

bbot/core/event/base.py

@@ -515,22 +515,25 @@ def scope_distance(self, scope_distance):
             new_scope_distance = min(self.scope_distance, scope_distance)
         if self._scope_distance != new_scope_distance:
             # remove old scope distance tags
-            for t in list(self.tags):
-                if t.startswith("distance-"):
-                    self.remove_tag(t)
-            if self.host:
-                if scope_distance == 0:
-                    self.add_tag("in-scope")
-                    self.remove_tag("affiliate")
-                else:
-                    self.remove_tag("in-scope")
-                    self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
+            self.refresh_scope_tags()
             # apply recursively to parent events
             parent_scope_distance = getattr(self.parent, "scope_distance", None)
             if parent_scope_distance is not None and self.parent is not self:
                 self.parent.scope_distance = new_scope_distance + 1
 
+    def refresh_scope_tags(self):
+        for t in list(self.tags):
+            if t.startswith("distance-"):
+                self.remove_tag(t)
+        if self.host:
+            if self.scope_distance == 0:
+                self.add_tag("in-scope")
+                self.remove_tag("affiliate")
+            else:
+                self.remove_tag("in-scope")
+                self.add_tag(f"distance-{self.scope_distance}")
+
     @property
     def scope_description(self):
         """
@@ -1352,18 +1355,22 @@ def sanitize_data(self, data):
         self.parsed_url = self.validators.validate_url_parsed(url)
         data["url"] = self.parsed_url.geturl()
 
-        header_dict = {}
-        for i in data.get("raw_header", "").splitlines():
-            if len(i) > 0 and ":" in i:
-                k, v = i.split(":", 1)
-                k = k.strip().lower()
-                v = v.lstrip()
-                if k in header_dict:
-                    header_dict[k].append(v)
-                else:
-                    header_dict[k] = [v]
+        if not "raw_header" in data:
+            raise ValueError("raw_header is required for HTTP_RESPONSE events")
+
+        if "header-dict" not in data:
+            header_dict = {}
+            for i in data.get("raw_header", "").splitlines():
+                if len(i) > 0 and ":" in i:
+                    k, v = i.split(":", 1)
+                    k = k.strip().lower()
+                    v = v.lstrip()
+                    if k in header_dict:
+                        header_dict[k].append(v)
+                    else:
+                        header_dict[k] = [v]
+            data["header-dict"] = header_dict
 
-        data["header-dict"] = header_dict
         # move URL to the front of the dictionary for visibility
         data = dict(data)
         new_data = {"url": data.pop("url")}
@@ -1377,6 +1384,13 @@ def _words(self):
     def _pretty_string(self):
         return f'{self.data["hash"]["header_mmh3"]}:{self.data["hash"]["body_mmh3"]}'
 
+    @property
+    def raw_response(self):
+        """
+        Formats the status code, headers, and body into a single string formatted as an HTTP/1.1 response.
+        """
+        return f'{self.data["raw_header"]}{self.data["body"]}'
+
     @property
     def http_status(self):
         try:

bbot/modules/dnsbrute_mutations.py

@@ -1,3 +1,5 @@
+import time
+
 from bbot.modules.base import BaseModule
 
 
@@ -40,8 +42,11 @@ async def handle_event(self, event):
                 except KeyError:
                     self.found[domain] = {subdomain}
 
-    def get_parent_event(self, subdomain):
-        parent_host = self.helpers.closest_match(subdomain, self.parent_events)
+    async def get_parent_event(self, subdomain):
+        start = time.time()
+        parent_host = await self.helpers.run_in_executor(self.helpers.closest_match, subdomain, self.parent_events)
+        elapsed = time.time() - start
+        self.trace(f"{subdomain}: got closest match among {len(self.parent_events):,} parent events in {elapsed:.2f}s")
         return self.parent_events[parent_host]
 
     async def finish(self):
@@ -124,7 +129,7 @@ def add_mutation(m):
                             self._mutation_run_counter[domain] = mutation_run = 1
                         self._mutation_run_counter[domain] += 1
                         for hostname in results:
-                            parent_event = self.get_parent_event(hostname)
+                            parent_event = await self.get_parent_event(hostname)
                             mutation_run_ordinal = self.helpers.integer_to_ordinal(mutation_run)
                             await self.emit_event(
                                 hostname,

bbot/modules/extractous.py

@@ -28,6 +28,7 @@ class extractous(BaseModule):
             "ica",  #  Citrix Independent Computing Architecture File
             "indd",  #  Adobe InDesign Document
             "ini",  #  Initialization File
+            "json",  #  JSON File
             "key",  #  Private Key File
             "pub",  #  Public Key File
             "log",  #  Log File
@@ -45,6 +46,7 @@ class extractous(BaseModule):
             "pptx",  #  Microsoft PowerPoint Presentation
             "ps1",  #  PowerShell Script
             "rdp",  #  Remote Desktop Protocol File
+            "rsa",  #  RSA Private Key File
             "sh",  #  Shell Script
             "sql",  #  SQL Database Dump
             "swp",  #  Swap File (temporary file, often Vim)

bbot/modules/filedownload.py

@@ -38,6 +38,7 @@ class filedownload(BaseModule):
             "indd",  #  Adobe InDesign Document
             "ini",  #  Initialization File
             "jar",  #  Java Archive
+            "json",  #  JSON File
             "key",  #  Private Key File
             "log",  #  Log File
             "markdown",  #  Markdown File
@@ -57,6 +58,7 @@ class filedownload(BaseModule):
             "pub",  #  Public Key File
             "raw",  #  Raw Image File Format
             "rdp",  #  Remote Desktop Protocol File
+            "rsa",  #  RSA Private Key File
             "sh",  #  Shell Script
             "sql",  #  SQL Database Dump
             "sqlite",  #  SQLite Database File

bbot/modules/trufflehog.py

@@ -3,7 +3,7 @@
 
 
 class trufflehog(BaseModule):
-    watched_events = ["CODE_REPOSITORY", "FILESYSTEM"]
+    watched_events = ["CODE_REPOSITORY", "FILESYSTEM", "HTTP_RESPONSE", "RAW_TEXT"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["passive", "safe", "code-enum"]
     meta = {
@@ -81,12 +81,15 @@ async def filter_event(self, event):
         return True
 
     async def handle_event(self, event):
-        description = event.data.get("description", "")
+        description = ""
+        if isinstance(event.data, dict):
+            description = event.data.get("description", "")
+
         if event.type == "CODE_REPOSITORY":
             path = event.data["url"]
             if "git" in event.tags:
                 module = "github-experimental"
-        else:
+        elif event.type == "FILESYSTEM":
             path = event.data["path"]
             if "git" in event.tags:
                 module = "git"
@@ -96,6 +99,14 @@ async def handle_event(self, event):
                 module = "postman"
             else:
                 module = "filesystem"
+        elif event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
+            module = "filesystem"
+            file_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data
+            # write the response to a tempfile
+            # this is necessary because trufflehog doesn't yet support reading from stdin
+            # https://github.com/trufflesecurity/trufflehog/issues/162
+            path = self.helpers.tempfile(file_data, pipe=False)
+
         if event.type == "CODE_REPOSITORY":
             host = event.host
         else:
@@ -108,41 +119,32 @@ async def handle_event(self, event):
             verified,
             source_metadata,
         ) in self.execute_trufflehog(module, path):
-            if verified:
-                data = {
-                    "severity": "High",
-                    "description": f"Verified Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
-                    "host": host,
-                }
-                if description:
-                    data["description"] += f" Description: [{description}]"
-                data["description"] += f" Raw result: [{raw_result}]"
-                if rawv2_result:
-                    data["description"] += f" RawV2 result: [{rawv2_result}]"
-                await self.emit_event(
-                    data,
-                    "VULNERABILITY",
-                    event,
-                    context=f'{{module}} searched {event.type} using "{module}" method and found verified secret ({{event.type}}): {raw_result}',
-                )
-            else:
-                data = {
-                    "description": f"Potential Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
-                    "host": host,
-                }
-                if description:
-                    data["description"] += f" Description: [{description}]"
-                data["description"] += f" Raw result: [{raw_result}]"
-                if rawv2_result:
-                    data["description"] += f" RawV2 result: [{rawv2_result}]"
-                await self.emit_event(
-                    data,
-                    "FINDING",
-                    event,
-                    context=f'{{module}} searched {event.type} using "{module}" method and found possible secret ({{event.type}}): {raw_result}',
-                )
-
-    async def execute_trufflehog(self, module, path):
+            verified_str = "Verified" if verified else "Possible"
+            finding_type = "VULNERABILITY" if verified else "FINDING"
+            data = {
+                "description": f"{verified_str} Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
+            }
+            if host:
+                data["host"] = host
+            if finding_type == "VULNERABILITY":
+                data["severity"] = "High"
+            if description:
+                data["description"] += f" Description: [{description}]"
+            data["description"] += f" Raw result: [{raw_result}]"
+            if rawv2_result:
+                data["description"] += f" RawV2 result: [{rawv2_result}]"
+            await self.emit_event(
+                data,
+                finding_type,
+                event,
+                context=f'{{module}} searched {event.type} using "{module}" method and found {verified_str.lower()} secret ({{event.type}}): {raw_result}',
+            )
+
+        # clean up the tempfile when we're done with it
+        if event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
+            path.unlink(missing_ok=True)
+
+    async def execute_trufflehog(self, module, path=None, string=None):
         command = [
             "trufflehog",
             "--json",

bbot/test/test_step_1/test_events.py

@@ -149,6 +149,7 @@ async def test_events(events, helpers):
             "title": "HTTP%20RESPONSE",
             "url": "http://www.evilcorp.com:80",
             "input": "http://www.evilcorp.com:80",
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
             "location": "/asdf",
             "status_code": 301,
         },
@@ -161,7 +162,13 @@ async def test_events(events, helpers):
 
     # http response url validation
     http_response_2 = scan.make_event(
-        {"port": "80", "url": "http://evilcorp.com:80/asdf"}, "HTTP_RESPONSE", dummy=True
+        {
+            "port": "80",
+            "url": "http://evilcorp.com:80/asdf",
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
+        },
+        "HTTP_RESPONSE",
+        dummy=True,
     )
     assert http_response_2.data["url"] == "http://evilcorp.com/asdf"
 
@@ -546,6 +553,10 @@ async def test_events(events, helpers):
     http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event)
     assert http_response.parent_id == scan.root_event.id
     assert http_response.data["input"] == "http://example.com:80"
+    assert (
+        http_response.raw_response
+        == 'HTTP/1.1 200 OK\r\nConnection: close\r\nAge: 526111\r\nCache-Control: max-age=604800\r\nContent-Type: text/html; charset=UTF-8\r\nDate: Mon, 14 Nov 2022 17:14:27 GMT\r\nEtag: "3147526947+ident+gzip"\r\nExpires: Mon, 21 Nov 2022 17:14:27 GMT\r\nLast-Modified: Thu, 17 Oct 2019 07:18:26 GMT\r\nServer: ECS (agb/A445)\r\nVary: Accept-Encoding\r\nX-Cache: HIT\r\n\r\n<!doctype html>\n<html>\n<head>\n    <title>Example Domain</title>\n\n    <meta charset="utf-8" />\n    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n    <meta name="viewport" content="width=device-width, initial-scale=1" />\n    <style type="text/css">\n    body {\n        background-color: #f0f0f2;\n        margin: 0;\n        padding: 0;\n        font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n        \n    }\n    div {\n        width: 600px;\n        margin: 5em auto;\n        padding: 2em;\n        background-color: #fdfdff;\n        border-radius: 0.5em;\n        box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);\n    }\n    a:link, a:visited {\n        color: #38488f;\n        text-decoration: none;\n    }\n    @media (max-width: 700px) {\n        div {\n            margin: 0 auto;\n            width: auto;\n        }\n    }\n    </style>    \n</head>\n\n<body>\n<div>\n    <h1>Example Domain</h1>\n    <p>This domain is for use in illustrative examples in documents. You may use this\n    domain in literature without prior coordination or asking for permission.</p>\n    <p><a href="https://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n'
+    )
     json_event = http_response.json(mode="graph")
     assert isinstance(json_event["data"], str)
     json_event = http_response.json()
@@ -906,7 +917,12 @@ def test_event_closest_host():
     assert event1.host == "evilcorp.com"
     # second event has a host + url
     event2 = scan.make_event(
-        {"method": "GET", "url": "http://www.evilcorp.com/asdf", "hash": {"header_mmh3": "1", "body_mmh3": "2"}},
+        {
+            "method": "GET",
+            "url": "http://www.evilcorp.com/asdf",
+            "hash": {"header_mmh3": "1", "body_mmh3": "2"},
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
+        },
         "HTTP_RESPONSE",
         parent=event1,
     )