diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 7ccae89e0..a0a569e53 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -93,7 +93,6 @@ "frolicking", "furry", "fuzzy", - "gay", "gentle", "giddy", "glowering", @@ -189,7 +188,6 @@ "psychic", "puffy", "pure", - "queer", "questionable", "rabid", "raging", @@ -276,6 +274,7 @@ "wispy", "witty", "woolly", + "zesty", ] names = [ diff --git a/bbot/modules/base.py b/bbot/modules/base.py index df4bed023..946506094 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -312,7 +312,7 @@ async def require_api_key(self): try: await self.ping() self.hugesuccess(f"API is ready") - return True + return True, "" except Exception as e: self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" @@ -331,7 +331,7 @@ def api_key(self, api_keys): self._api_keys = list(api_keys) def cycle_api_key(self): - if self._api_keys: + if len(self._api_keys) > 1: self.verbose(f"Cycling API key") self._api_keys.insert(0, self._api_keys.pop()) else: @@ -345,25 +345,42 @@ def api_retries(self): def api_failure_abort_threshold(self): return (self.api_retries * self._api_failure_abort_threshold) + 1 - async def ping(self): + async def ping(self, url=None): """Asynchronously checks the health of the configured API. - This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. This method should include an assert statement to validate the API's health, typically by making a test request to a known endpoint. + This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. It makes a test request to a known endpoint to validate the API's health. - Example Usage: - In your implementation, if the API has a "/ping" endpoint: - async def ping(self): - r = await self.api_request(f"{self.base_url}/ping") - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + The method uses the `ping_url` attribute if defined, or falls back to a provided URL. If neither is available, no request is made. + + Args: + url (str, optional): A specific URL to use for the ping request. If not provided, the method will use the `ping_url` attribute. Returns: None Raises: - AssertionError: If the API does not respond as expected. - """ - return + ValueError: If the API response is not successful (status code != 200). + + Example Usage: + To use this method, simply define the `ping_url` attribute in your module: + + class MyModule(BaseModule): + ping_url = "https://api.example.com/ping" + + Alternatively, you can override this method for more complex health checks: + + async def ping(self): + r = await self.api_request(f"{self.base_url}/complex-health-check") + if r.status_code != 200 or r.json().get('status') != 'healthy': + raise ValueError(f"API unhealthy: {r.text}") + """ + if url is None: + url = getattr(self, "ping_url", "") + if url: + r = await self.api_request(url) + if getattr(r, "status_code", 0) != 200: + response_text = getattr(r, "text", "no response from server") + raise ValueError(response_text) @property def batch_size(self): @@ -1134,6 +1151,8 @@ async def api_request(self, *args, **kwargs): self._api_request_failures = 0 else: status_code = getattr(r, "status_code", 0) + response_text = getattr(r, "text", "") + self.trace(f"API response to {url} failed with status code {status_code}: {response_text}") self._api_request_failures += 1 if self._api_request_failures >= self.api_failure_abort_threshold: self.set_error_state( diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 50e891811..f3889e7fd 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -29,9 +29,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["X-Access-Token"] = self.api_key return url, kwargs - async def ping(self): - pass - async def handle_event(self, event): query = self.make_query(event) subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains) diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 76f8a4e39..19e880034 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -27,10 +27,6 @@ class builtwith(subdomain_enum_apikey): options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"} base_url = "https://api.builtwith.com" - async def ping(self): - # builtwith does not have a ping feature, so we skip it to save API credits - return - async def handle_event(self, event): query = self.make_query(event) # domains diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 062b5523e..7e703966b 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -15,11 +15,12 @@ class c99(subdomain_enum_apikey): options_desc = {"api_key": "c99.nl API key"} base_url = "https://api.c99.nl" + ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json" async def ping(self): url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json" response = await self.api_request(url) - assert response.json()["success"] == True + assert response.json()["success"] == True, getattr(response, "text", "no response from server") async def request_url(self, query): url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json" diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py index ecc960690..cba4e7ea4 100644 --- a/bbot/modules/chaos.py +++ b/bbot/modules/chaos.py @@ -15,11 +15,7 @@ class chaos(subdomain_enum_apikey): options_desc = {"api_key": "Chaos API key"} base_url = "https://dns.projectdiscovery.io/dns" - - async def ping(self): - url = f"{self.base_url}/example.com" - response = await self.api_request(url) - assert response.json()["domain"] == "example.com" + ping_url = f"{base_url}/example.com" def prepare_api_request(self, url, kwargs): kwargs["headers"]["Authorization"] = self.api_key diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index f5a275b41..8977ddfbe 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -15,14 +15,9 @@ class hunterio(subdomain_enum_apikey): options_desc = {"api_key": "Hunter.IO API key"} base_url = "https://api.hunter.io/v2" + ping_url = f"{base_url}/account?api_key={{api_key}}" limit = 100 - async def ping(self): - url = f"{self.base_url}/account?api_key={{api_key}}" - r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content - async def handle_event(self, event): query = self.make_query(event) for entry in await self.query(query): diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index 4118d3471..2a4b387f4 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -32,9 +32,7 @@ async def setup(self): async def ping(self): url = self.build_url("8.8.8.8") - r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + await super().ping(url) def build_url(self, data): url = f"{self.base_url}/?key={{api_key}}&ip={data}&format=json&source=bbot" diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index f3caf77f0..02cfe0f3d 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -23,16 +23,11 @@ class Ipstack(BaseModule): suppress_dupes = False base_url = "http://api.ipstack.com" + ping_url = f"{base_url}/check?access_key={{api_key}}" async def setup(self): return await self.require_api_key() - async def ping(self): - url = f"{self.base_url}/check?access_key={{api_key}}" - r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content - async def handle_event(self, event): try: url = f"{self.base_url}/{event.data}?access_key={{api_key}}" diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index bb63abd80..22be7513d 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -15,6 +15,7 @@ class leakix(subdomain_enum_apikey): } base_url = "https://leakix.net" + ping_url = f"{base_url}/host/1.2.3.4.5" async def setup(self): ret = await super(subdomain_enum_apikey, self).setup() @@ -30,12 +31,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["api-key"] = self.api_key return url, kwargs - async def ping(self): - url = f"{self.base_url}/host/1.2.3.4.5" - r = await self.helpers.request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) != 401, resp_content - async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" response = await self.api_request(url) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 6222e41c0..104f4261b 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -30,11 +30,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["X-Api-Key"] = self.api_key return url, kwargs - async def ping(self): - url = f"{self.api_url}/me" - response = await self.api_request(url) - assert getattr(response, "status_code", 0) == 200, response.text - async def filter_event(self, event): if event.type == "CODE_REPOSITORY": if "postman" not in event.tags: diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index 8a24ed1b6..c74450307 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -15,17 +15,12 @@ class securitytrails(subdomain_enum_apikey): options_desc = {"api_key": "SecurityTrails API key"} base_url = "https://api.securitytrails.com/v1" + ping_url = f"{base_url}/ping?apikey={{api_key}}" async def setup(self): self.limit = 100 return await super().setup() - async def ping(self): - url = f"{self.base_url}/ping?apikey={{api_key}}" - r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content - async def request_url(self, query): url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}" response = await self.api_request(url) diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py new file mode 100644 index 000000000..16d2a564f --- /dev/null +++ b/bbot/modules/subdomainradar.py @@ -0,0 +1,160 @@ +import time +import asyncio + +from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey + + +class SubdomainRadar(subdomain_enum_apikey): + watched_events = ["DNS_NAME"] + produced_events = ["DNS_NAME"] + flags = ["subdomain-enum", "passive", "safe"] + meta = { + "description": "Query the Subdomain API for subdomains", + "created_date": "2022-07-08", + "author": "@TheTechromancer", + "auth_required": True, + } + options = {"api_key": "", "group": "fast", "timeout": 120} + options_desc = { + "api_key": "SubDomainRadar.io API key", + "group": "The enumeration group to use. Choose from fast, medium, deep", + "timeout": "Timeout in seconds", + } + + base_url = "https://api.subdomainradar.io" + ping_url = f"{base_url}/profile" + group_choices = ("fast", "medium", "deep") + + # set this really high so the poll loop finishes as soon as possible + _qsize = 9999999 + + async def setup(self): + self.group = self.config.get("group", "fast").strip().lower() + self.timeout = self.config.get("timeout", 120) + if self.group not in self.group_choices: + return False, f'Invalid group: "{self.group}", please choose from {",".join(self.group_choices)}' + success, reason = await self.require_api_key() + if not success: + return success, reason + # convert groups to enumerators + enumerators = {} + response = await self.api_request(f"{self.base_url}/enumerators/groups") + status_code = getattr(response, "status_code", 0) + if status_code != 200: + return False, f"Failed to get enumerators: (HTTP status code: {status_code})" + else: + try: + j = response.json() + except Exception: + return False, f"Failed to get enumerators: failed to parse response as JSON" + for group in j: + group_name = group.get("name", "").strip().lower() + if group_name: + group_enumerators = [] + for enumerator in group.get("enumerators", []): + enumerator_name = enumerator.get("display_name", "") + if enumerator_name: + group_enumerators.append(enumerator_name) + if group_enumerators: + enumerators[group_name] = group_enumerators + + self.enumerators = enumerators.get(self.group, []) + if not self.enumerators: + return False, f'No enumerators found for group: "{self.group}" ({self.enumerators})' + + self.enum_tasks = {} + self.poll_task = asyncio.create_task(self.task_poll_loop()) + + return True + + def prepare_api_request(self, url, kwargs): + if self.api_key: + kwargs["headers"] = {"Authorization": f"Bearer {self.api_key}"} + return url, kwargs + + async def handle_event(self, event): + query = self.make_query(event) + # start enumeration task + url = f"{self.base_url}/enumerate" + response = await self.api_request( + url, method="POST", json={"domains": [query], "enumerators": self.enumerators} + ) + try: + j = response.json() + except Exception: + self.warning(f"Failed to parse response as JSON: {getattr(response, 'text', '')}") + return + task_id = j.get("tasks", {}).get(query, "") + if not task_id: + self.warning(f"Failed to start enumeration for {query}") + return + self.enum_tasks[query] = (task_id, time.time(), event) + self.debug(f"Started enumeration task for {query}; task id: {task_id}") + + async def task_poll_loop(self): + # async with self._task_counter.count(f"{self.name}.task_poll_loop()"): + while 1: + for query, (task_id, start_time, event) in list(self.enum_tasks.items()): + url = f"{self.base_url}/tasks/{task_id}" + response = await self.api_request(url) + if getattr(response, "status_code", 0) == 200: + finished = await self.parse_response(response, query, event) + if finished: + self.enum_tasks.pop(query) + continue + # if scan is finishing, consider timeout + if self.scan.status == "FINISHING": + if start_time + self.timeout < time.time(): + self.enum_tasks.pop(query) + self.info(f"Enumeration task for {query} timed out") + + if self.scan.status == "FINISHING" and not self.enum_tasks: + break + await self.helpers.sleep(5) + + async def parse_response(self, response, query, event): + j = response.json() + status = j.get("status", "") + if status.lower() == "completed": + for subdomain in j.get("subdomains", []): + hostname = subdomain.get("subdomain", "") + if hostname and hostname.endswith(f".{query}") and not hostname == event.data: + await self.emit_event( + hostname, + "DNS_NAME", + event, + abort_if=self.abort_if, + context=f'{{module}} searched SubDomainRadar.io API for "{query}" and found {{event.type}}: {{event.data}}', + ) + return True + return False + + async def finish(self): + start_time = time.time() + while self.enum_tasks and not self.poll_task.done(): + elapsed_time = time.time() - start_time + if elapsed_time >= self.timeout: + self.warning(f"Timed out waiting for the following tasks to finish: {self.enum_tasks}") + for query, (task_id, _, _) in list(self.enum_tasks.items()): + url = f"{self.base_url}/tasks/{task_id}" + self.warning(f" - {query} ({url})") + break + + self.verbose( + f"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)" + ) + + try: + # Wait for the task to complete or for 10 seconds, whichever comes first + await asyncio.wait_for(asyncio.shield(self.poll_task), timeout=10) + except asyncio.TimeoutError: + # This just means our 10-second check has elapsed, not that the task failed + pass + + # Cancel the poll_task if it's still running + if not self.poll_task.done(): + self.poll_task.cancel() + try: + await self.poll_task + except asyncio.CancelledError: + pass diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py index 35c68a211..6769d00ab 100644 --- a/bbot/modules/templates/github.py +++ b/bbot/modules/templates/github.py @@ -11,6 +11,7 @@ class github(BaseModule): _qsize = 1 base_url = "https://api.github.com" + ping_url = f"{base_url}/zen" def prepare_api_request(self, url, kwargs): kwargs["headers"]["Authorization"] = f"token {self.api_key}" @@ -20,15 +21,17 @@ async def setup(self): await super().setup() self.headers = {} api_keys = set() - for module_name in ("github", "github_codesearch", "github_org", "git_clone"): - module_config = self.scan.config.get("modules", {}).get(module_name, {}) + modules_config = self.scan.config.get("modules", {}) + git_modules = [m for m in modules_config if str(m).startswith("git")] + for module_name in git_modules: + module_config = modules_config.get(module_name, {}) api_key = module_config.get("api_key", "") if isinstance(api_key, str): api_key = [api_key] for key in api_key: key = key.strip() if key: - api_keys.update(key) + api_keys.add(key) if not api_keys: if self.auth_required: return None, "No API key set" @@ -41,8 +44,3 @@ async def setup(self): self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" return True - - async def ping(self): - url = f"{self.base_url}/zen" - response = await self.helpers.request(url, headers=self.headers) - assert getattr(response, "status_code", 0) == 200, response.text diff --git a/bbot/modules/templates/postman.py b/bbot/modules/templates/postman.py index 38cc3d04b..2048abc32 100644 --- a/bbot/modules/templates/postman.py +++ b/bbot/modules/templates/postman.py @@ -10,6 +10,7 @@ class postman(BaseModule): base_url = "https://www.postman.com/_api" api_url = "https://api.getpostman.com" html_url = "https://www.postman.com" + ping_url = f"{api_url}/me" headers = { "Content-Type": "application/json", diff --git a/bbot/modules/templates/shodan.py b/bbot/modules/templates/shodan.py index 3cc84022d..536046439 100644 --- a/bbot/modules/templates/shodan.py +++ b/bbot/modules/templates/shodan.py @@ -8,6 +8,7 @@ class shodan(subdomain_enum): options_desc = {"api_key": "Shodan API key"} base_url = "https://api.shodan.io" + ping_url = f"{base_url}/api-info?key={{api_key}}" async def setup(self): await super().setup() @@ -32,10 +33,3 @@ async def setup(self): except Exception as e: self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" - return True - - async def ping(self): - url = f"{self.base_url}/api-info?key={self.api_key}" - r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index e161636b2..95a040b1c 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -20,8 +20,8 @@ class subdomain_enum(BaseModule): # whether to reject wildcard DNS_NAMEs reject_wildcards = "strict" - # set qsize to 10. this helps combat rate limiting by ensuring that a query doesn't execute - # until the queue is ready to receive its results + # set qsize to 10. this helps combat rate limiting by ensuring the next query doesn't execute + # until the result from the previous queue have been consumed by the scan # we don't use 1 because it causes delays due to the asyncio.sleep; 10 gives us reasonable buffer room _qsize = 10 diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index 33b4d672c..c17aa6160 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -19,6 +19,7 @@ class Trickest(subdomain_enum_apikey): } base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be" + ping_url = f"{base_url}/dataset" dataset_id = "a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc" page_size = 50 @@ -26,15 +27,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["Authorization"] = f"Token {self.api_key}" return url, kwargs - async def ping(self): - url = f"{self.base_url}/dataset" - response = await self.api_request(url) - status_code = getattr(response, "status_code", 0) - if status_code != 200: - response_text = getattr(response, "text", "no response from server") - return False, response_text - return True - async def handle_event(self, event): query = self.make_query(event) async for result_batch in self.query(query): diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index 87e82766d..98f469e39 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -16,10 +16,6 @@ class virustotal(subdomain_enum_apikey): base_url = "https://www.virustotal.com/api/v3" - async def ping(self): - # virustotal does not have a ping function - return - def prepare_api_request(self, url, kwargs): kwargs["headers"]["x-apikey"] = self.api_key return url, kwargs diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 85b9a0073..35cbaf220 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -444,7 +444,6 @@ async def _mark_finished(self): # wait until output modules are flushed while 1: modules_finished = all([m.finished for m in output_modules]) - self.verbose(modules_finished) if modules_finished: break await asyncio.sleep(0.05) diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py index 039c6125b..d8003fd2a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_github_org.py +++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py @@ -10,9 +10,12 @@ async def setup_before_prep(self, module_test): {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}} ) - module_test.httpx_mock.add_response(url="https://api.github.com/zen") + module_test.httpx_mock.add_response( + url="https://api.github.com/zen", match_headers={"Authorization": "token asdf"} + ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity", + match_headers={"Authorization": "token asdf"}, json={ "login": "blacklanternsecurity", "id": 25311592, @@ -48,6 +51,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity/repos?per_page=100&page=1", + match_headers={"Authorization": "token asdf"}, json=[ { "id": 459780477, @@ -154,6 +158,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity/members?per_page=100&page=1", + match_headers={"Authorization": "token asdf"}, json=[ { "login": "TheTechromancer", @@ -179,6 +184,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.github.com/users/TheTechromancer/repos?per_page=100&page=1", + match_headers={"Authorization": "token asdf"}, json=[ { "id": 688270318, @@ -332,7 +338,7 @@ def check(self, module_test, events): class TestGithub_Org_No_Members(TestGithub_Org): - config_overrides = {"modules": {"github_org": {"include_members": False}}} + config_overrides = {"modules": {"github_org": {"include_members": False}, "github": {"api_key": "asdf"}}} def check(self, module_test, events): assert len(events) == 6 @@ -360,7 +366,7 @@ def check(self, module_test, events): class TestGithub_Org_MemberRepos(TestGithub_Org): - config_overrides = {"modules": {"github_org": {"include_member_repos": True}}} + config_overrides = {"modules": {"github_org": {"include_member_repos": True}, "github": {"api_key": "asdf"}}} def check(self, module_test, events): assert len(events) == 8 @@ -378,7 +384,12 @@ def check(self, module_test, events): class TestGithub_Org_Custom_Target(TestGithub_Org): targets = ["ORG:blacklanternsecurity"] - config_overrides = {"scope": {"report_distance": 10}, "omit_event_types": [], "speculate": True} + config_overrides = { + "scope": {"report_distance": 10}, + "omit_event_types": [], + "speculate": True, + "modules": {"github": {"api_key": "asdf"}}, + } def check(self, module_test, events): assert len(events) == 8 diff --git a/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py b/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py new file mode 100644 index 000000000..c2bb827f3 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py @@ -0,0 +1,208 @@ +from .base import ModuleTestBase + + +class TestSubDomainRadar(ModuleTestBase): + config_overrides = {"modules": {"subdomainradar": {"api_key": "asdf"}}} + + async def setup_before_prep(self, module_test): + await module_test.mock_dns( + { + "blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + "www.blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + "asdf.blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + } + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/profile", + match_headers={"Authorization": "Bearer asdf"}, + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/enumerate", + method="POST", + json={ + "tasks": {"blacklanternsecurity.com": "86de4531-0a67-41fe-b5e4-8ce8207d6245"}, + "message": "Tasks initiated", + }, + match_headers={"Authorization": "Bearer asdf"}, + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/tasks/86de4531-0a67-41fe-b5e4-8ce8207d6245", + match_headers={"Authorization": "Bearer asdf"}, + json={ + "task_id": "86de4531-0a67-41fe-b5e4-8ce8207d6245", + "status": "completed", + "domain": "blacklanternsecurity.com", + "subdomains": [ + { + "subdomain": "www.blacklanternsecurity.com", + "ip": None, + "reverse_dns": [], + "country": None, + "timestamp": None, + }, + { + "subdomain": "asdf.blacklanternsecurity.com", + "ip": None, + "reverse_dns": [], + "country": None, + "timestamp": None, + }, + ], + "total_subdomains": 2, + "rank": None, + "whois": { + "domain_name": ["BLACKLANTERNSECURITY.COM", "blacklanternsecurity.com"], + "registrar": "MarkMonitor, Inc.", + "creation_date": ["1992-11-04T05:00:00", "1992-11-04T05:00:00+00:00"], + "expiration_date": ["2026-11-03T05:00:00", "2026-11-03T00:00:00+00:00"], + "last_updated": ["2024-10-02T10:15:20", "2024-10-02T10:15:20+00:00"], + "status": [ + "clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited", + "clientTransferProhibited https://icann.org/epp#clientTransferProhibited", + "clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited", + "serverDeleteProhibited https://icann.org/epp#serverDeleteProhibited", + "serverTransferProhibited https://icann.org/epp#serverTransferProhibited", + "serverUpdateProhibited https://icann.org/epp#serverUpdateProhibited", + "clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)", + "clientTransferProhibited (https://www.icann.org/epp#clientTransferProhibited)", + "clientDeleteProhibited (https://www.icann.org/epp#clientDeleteProhibited)", + "serverUpdateProhibited (https://www.icann.org/epp#serverUpdateProhibited)", + "serverTransferProhibited (https://www.icann.org/epp#serverTransferProhibited)", + "serverDeleteProhibited (https://www.icann.org/epp#serverDeleteProhibited)", + ], + "nameservers": [ + "A1-12.AKAM.NET", + "A10-67.AKAM.NET", + "A12-64.AKAM.NET", + "A28-65.AKAM.NET", + "A7-66.AKAM.NET", + "A9-67.AKAM.NET", + "EDNS69.ULTRADNS.BIZ", + "EDNS69.ULTRADNS.COM", + "EDNS69.ULTRADNS.NET", + "EDNS69.ULTRADNS.ORG", + "edns69.ultradns.biz", + "a12-64.akam.net", + "edns69.ultradns.net", + "edns69.ultradns.org", + "a10-67.akam.net", + "a28-65.akam.net", + "a9-67.akam.net", + "a1-12.akam.net", + "a7-66.akam.net", + "edns69.ultradns.com", + ], + "emails": [ + "abusecomplaints@markmonitor.com", + "admin@dnstinations.com", + "whoisrequest@markmonitor.com", + ], + "dnssec": "unsigned", + "org": "DNStination Inc.", + "address": "3450 Sacramento Street, Suite 405", + "city": "San Francisco", + "state": "CA", + "zipcode": None, + "country": "US", + }, + "enumerators": ["Aquarius Enumerator", "Beta Enumerator", "Chi Enumerator", "Eta Enumerator"], + "timestamp": "2024-10-06T02:48:10.075636", + "error": None, + "is_notification": False, + "notification_domain_id": None, + "demo": False, + "user_id": 49, + "time_to_finish": 41, + }, + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/enumerators/groups", + match_headers={"Authorization": "Bearer asdf"}, + json=[ + { + "id": "1", + "name": "Fast", + "description": "Enumerators optimized for high-speed scanning and rapid data collection", + "enumerators": [ + {"display_name": "Beta Enumerator"}, + {"display_name": "Chi Enumerator"}, + {"display_name": "Aquarius Enumerator"}, + {"display_name": "Eta Enumerator"}, + ], + }, + { + "id": "2", + "name": "Medium", + "description": "Enumerators balanced for moderate speed with a focus on thoroughness", + "enumerators": [ + {"display_name": "Kappa Enumerator"}, + {"display_name": "Lambda Enumerator"}, + {"display_name": "Mu Enumerator"}, + {"display_name": "Pi Enumerator"}, + {"display_name": "Tau Enumerator"}, + {"display_name": "Beta Enumerator"}, + {"display_name": "Chi Enumerator"}, + {"display_name": "Psi Enumerator"}, + {"display_name": "Aquarius Enumerator"}, + {"display_name": "Zeta Enumerator"}, + {"display_name": "Eta Enumerator"}, + ], + }, + { + "id": "3", + "name": "Deep", + "description": "Enumerators designed for exhaustive searches and in-depth data analysis", + "enumerators": [ + {"display_name": "Alpha Enumerator"}, + {"display_name": "Kappa Enumerator"}, + {"display_name": "Lambda Enumerator"}, + {"display_name": "Mu Enumerator"}, + {"display_name": "Nu Enumerator"}, + {"display_name": "Xi Enumerator"}, + {"display_name": "Pi Enumerator"}, + {"display_name": "Rho Enumerator"}, + {"display_name": "Sigma Enumerator"}, + {"display_name": "Tau Enumerator"}, + {"display_name": "Beta Enumerator"}, + {"display_name": "Chi Enumerator"}, + {"display_name": "Omega Enumerator"}, + {"display_name": "Psi Enumerator"}, + {"display_name": "Phi Enumerator"}, + {"display_name": "Axon Enumerator"}, + {"display_name": "Aquarius Enumerator"}, + {"display_name": "Pegasus Enumerator"}, + {"display_name": "Petra Enumerator"}, + {"display_name": "Oasis Enumerator"}, + {"display_name": "Mike Enumerator"}, + {"display_name": "Cat Enumerator"}, + {"display_name": "Brutus Enumerator"}, + {"display_name": "Dee Enumerator"}, + {"display_name": "Jul Enumerator"}, + {"display_name": "Eve Enumerator"}, + {"display_name": "Frank Enumerator"}, + {"display_name": "Gus Enumerator"}, + {"display_name": "Hank Enumerator"}, + {"display_name": "Delta Enumerator"}, + {"display_name": "Ivy Enumerator"}, + {"display_name": "Jack Enumerator"}, + {"display_name": "Karl Enumerator"}, + {"display_name": "Liam Enumerator"}, + {"display_name": "Nora Enumerator"}, + {"display_name": "Mars Enumerator"}, + {"display_name": "Neptune Enumerator"}, + {"display_name": "Orion Enumerator"}, + {"display_name": "Oedipus Enumerator"}, + {"display_name": "Pandora Enumerator"}, + {"display_name": "Epsilon Enumerator"}, + {"display_name": "Zeta Enumerator"}, + {"display_name": "Eta Enumerator"}, + {"display_name": "Theta Enumerator"}, + {"display_name": "Iota Enumerator"}, + ], + }, + ], + ) + + def check(self, module_test, events): + assert any(e.data == "www.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #1" + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #2"