Merge pull request #1824 from blacklanternsecurity/subdomainradar.io

Add SubdomainRadar.io Module

bbot/core/helpers/names_generator.py

@@ -93,7 +93,6 @@
     "frolicking",
     "furry",
     "fuzzy",
-    "gay",
     "gentle",
     "giddy",
     "glowering",
@@ -189,7 +188,6 @@
     "psychic",
     "puffy",
     "pure",
-    "queer",
     "questionable",
     "rabid",
     "raging",
@@ -276,6 +274,7 @@
     "wispy",
     "witty",
     "woolly",
+    "zesty",
 ]
 
 names = [

bbot/modules/base.py

@@ -312,7 +312,7 @@ async def require_api_key(self):
             try:
                 await self.ping()
                 self.hugesuccess(f"API is ready")
-                return True
+                return True, ""
             except Exception as e:
                 self.trace(traceback.format_exc())
                 return None, f"Error with API ({str(e).strip()})"
@@ -331,7 +331,7 @@ def api_key(self, api_keys):
         self._api_keys = list(api_keys)
 
     def cycle_api_key(self):
-        if self._api_keys:
+        if len(self._api_keys) > 1:
             self.verbose(f"Cycling API key")
             self._api_keys.insert(0, self._api_keys.pop())
         else:
@@ -345,25 +345,42 @@ def api_retries(self):
     def api_failure_abort_threshold(self):
         return (self.api_retries * self._api_failure_abort_threshold) + 1
 
-    async def ping(self):
+    async def ping(self, url=None):
         """Asynchronously checks the health of the configured API.
 
-        This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. This method should include an assert statement to validate the API's health, typically by making a test request to a known endpoint.
+        This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. It makes a test request to a known endpoint to validate the API's health.
 
-        Example Usage:
-            In your implementation, if the API has a "/ping" endpoint:
-            async def ping(self):
-                r = await self.api_request(f"{self.base_url}/ping")
-                resp_content = getattr(r, "text", "")
-                assert getattr(r, "status_code", 0) == 200, resp_content
+        The method uses the `ping_url` attribute if defined, or falls back to a provided URL. If neither is available, no request is made.
+
+        Args:
+            url (str, optional): A specific URL to use for the ping request. If not provided, the method will use the `ping_url` attribute.
 
         Returns:
             None
 
         Raises:
-            AssertionError: If the API does not respond as expected.
-        """
-        return
+            ValueError: If the API response is not successful (status code != 200).
+
+        Example Usage:
+            To use this method, simply define the `ping_url` attribute in your module:
+
+            class MyModule(BaseModule):
+                ping_url = "https://api.example.com/ping"
+
+            Alternatively, you can override this method for more complex health checks:
+
+            async def ping(self):
+                r = await self.api_request(f"{self.base_url}/complex-health-check")
+                if r.status_code != 200 or r.json().get('status') != 'healthy':
+                    raise ValueError(f"API unhealthy: {r.text}")
+        """
+        if url is None:
+            url = getattr(self, "ping_url", "")
+        if url:
+            r = await self.api_request(url)
+            if getattr(r, "status_code", 0) != 200:
+                response_text = getattr(r, "text", "no response from server")
+                raise ValueError(response_text)
 
     @property
     def batch_size(self):
@@ -1134,6 +1151,8 @@ async def api_request(self, *args, **kwargs):
                 self._api_request_failures = 0
             else:
                 status_code = getattr(r, "status_code", 0)
+                response_text = getattr(r, "text", "")
+                self.trace(f"API response to {url} failed with status code {status_code}: {response_text}")
                 self._api_request_failures += 1
                 if self._api_request_failures >= self.api_failure_abort_threshold:
                     self.set_error_state(

bbot/modules/bevigil.py

@@ -29,9 +29,6 @@ def prepare_api_request(self, url, kwargs):
         kwargs["headers"]["X-Access-Token"] = self.api_key
         return url, kwargs
 
-    async def ping(self):
-        pass
-
     async def handle_event(self, event):
         query = self.make_query(event)
         subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains)

bbot/modules/builtwith.py

@@ -27,10 +27,6 @@ class builtwith(subdomain_enum_apikey):
     options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"}
     base_url = "https://api.builtwith.com"
 
-    async def ping(self):
-        # builtwith does not have a ping feature, so we skip it to save API credits
-        return
-
     async def handle_event(self, event):
         query = self.make_query(event)
         # domains

bbot/modules/c99.py

@@ -15,11 +15,12 @@ class c99(subdomain_enum_apikey):
     options_desc = {"api_key": "c99.nl API key"}
 
     base_url = "https://api.c99.nl"
+    ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json"
 
     async def ping(self):
         url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json"
         response = await self.api_request(url)
-        assert response.json()["success"] == True
+        assert response.json()["success"] == True, getattr(response, "text", "no response from server")
 
     async def request_url(self, query):
         url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json"

bbot/modules/chaos.py

@@ -15,11 +15,7 @@ class chaos(subdomain_enum_apikey):
     options_desc = {"api_key": "Chaos API key"}
 
     base_url = "https://dns.projectdiscovery.io/dns"
-
-    async def ping(self):
-        url = f"{self.base_url}/example.com"
-        response = await self.api_request(url)
-        assert response.json()["domain"] == "example.com"
+    ping_url = f"{base_url}/example.com"
 
     def prepare_api_request(self, url, kwargs):
         kwargs["headers"]["Authorization"] = self.api_key

bbot/modules/hunterio.py

@@ -15,14 +15,9 @@ class hunterio(subdomain_enum_apikey):
     options_desc = {"api_key": "Hunter.IO API key"}
 
     base_url = "https://api.hunter.io/v2"
+    ping_url = f"{base_url}/account?api_key={{api_key}}"
     limit = 100
 
-    async def ping(self):
-        url = f"{self.base_url}/account?api_key={{api_key}}"
-        r = await self.api_request(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
-
     async def handle_event(self, event):
         query = self.make_query(event)
         for entry in await self.query(query):

bbot/modules/ip2location.py

@@ -32,9 +32,7 @@ async def setup(self):
 
     async def ping(self):
         url = self.build_url("8.8.8.8")
-        r = await self.api_request(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
+        await super().ping(url)
 
     def build_url(self, data):
         url = f"{self.base_url}/?key={{api_key}}&ip={data}&format=json&source=bbot"

bbot/modules/ipstack.py

@@ -23,16 +23,11 @@ class Ipstack(BaseModule):
     suppress_dupes = False
 
     base_url = "http://api.ipstack.com"
+    ping_url = f"{base_url}/check?access_key={{api_key}}"
 
     async def setup(self):
         return await self.require_api_key()
 
-    async def ping(self):
-        url = f"{self.base_url}/check?access_key={{api_key}}"
-        r = await self.api_request(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
-
     async def handle_event(self, event):
         try:
             url = f"{self.base_url}/{event.data}?access_key={{api_key}}"

bbot/modules/leakix.py

@@ -15,6 +15,7 @@ class leakix(subdomain_enum_apikey):
     }
 
     base_url = "https://leakix.net"
+    ping_url = f"{base_url}/host/1.2.3.4.5"
 
     async def setup(self):
         ret = await super(subdomain_enum_apikey, self).setup()
@@ -30,12 +31,6 @@ def prepare_api_request(self, url, kwargs):
             kwargs["headers"]["api-key"] = self.api_key
         return url, kwargs
 
-    async def ping(self):
-        url = f"{self.base_url}/host/1.2.3.4.5"
-        r = await self.helpers.request(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) != 401, resp_content
-
     async def request_url(self, query):
         url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}"
         response = await self.api_request(url)

bbot/modules/postman_download.py

@@ -30,11 +30,6 @@ def prepare_api_request(self, url, kwargs):
         kwargs["headers"]["X-Api-Key"] = self.api_key
         return url, kwargs
 
-    async def ping(self):
-        url = f"{self.api_url}/me"
-        response = await self.api_request(url)
-        assert getattr(response, "status_code", 0) == 200, response.text
-
     async def filter_event(self, event):
         if event.type == "CODE_REPOSITORY":
             if "postman" not in event.tags:

bbot/modules/securitytrails.py

@@ -15,17 +15,12 @@ class securitytrails(subdomain_enum_apikey):
     options_desc = {"api_key": "SecurityTrails API key"}
 
     base_url = "https://api.securitytrails.com/v1"
+    ping_url = f"{base_url}/ping?apikey={{api_key}}"
 
     async def setup(self):
         self.limit = 100
         return await super().setup()
 
-    async def ping(self):
-        url = f"{self.base_url}/ping?apikey={{api_key}}"
-        r = await self.api_request(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
-
     async def request_url(self, query):
         url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}"
         response = await self.api_request(url)