Merge pull request #1944 from blacklanternsecurity/mysql

New Module: MySQL Output

README.md

@@ -240,6 +240,24 @@ Click the graph below to explore the [inner workings](https://www.blacklanternse
 
 [![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)
 
+## Output Modules
+
+- [Neo4j](docs/scanning/output.md#neo4j)
+- [Teams](docs/scanning/output.md#teams)
+- [Discord](docs/scanning/output.md#discord)
+- [Slack](docs/scanning/output.md#slack)
+- [Postgres](docs/scanning/output.md#postgres)
+- [MySQL](docs/scanning/output.md#mysql)
+- [SQLite](docs/scanning/output.md#sqlite)
+- [Splunk](docs/scanning/output.md#splunk)
+- [Elasticsearch](docs/scanning/output.md#elasticsearch)
+- [CSV](docs/scanning/output.md#csv)
+- [JSON](docs/scanning/output.md#json)
+- [HTTP](docs/scanning/output.md#http)
+- [Websocket](docs/scanning/output.md#websocket)
+
+...and [more](docs/scanning/output.md)!
+
 ## BBOT as a Python Library
 
 #### Synchronous

bbot/db/sql/models.py

@@ -141,8 +141,8 @@ class Target(BBOTBaseModel, table=True):
     seeds: List = Field(default=[], sa_type=JSON)
     whitelist: List = Field(default=None, sa_type=JSON)
     blacklist: List = Field(default=[], sa_type=JSON)
-    hash: str = Field(sa_column=Column("hash", String, unique=True, primary_key=True, index=True))
-    scope_hash: str = Field(sa_column=Column("scope_hash", String, index=True))
-    seed_hash: str = Field(sa_column=Column("seed_hashhash", String, index=True))
-    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String, index=True))
-    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String, index=True))
+    hash: str = Field(sa_column=Column("hash", String(length=255), unique=True, primary_key=True, index=True))
+    scope_hash: str = Field(sa_column=Column("scope_hash", String(length=255), index=True))
+    seed_hash: str = Field(sa_column=Column("seed_hashhash", String(length=255), index=True))
+    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String(length=255), index=True))
+    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String(length=255), index=True))

bbot/modules/output/mysql.py

@@ -0,0 +1,51 @@
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class MySQL(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a MySQL database"}
+    options = {
+        "username": "root",
+        "password": "bbotislife",
+        "host": "localhost",
+        "port": 3306,
+        "database": "bbot",
+    }
+    options_desc = {
+        "username": "The username to connect to MySQL",
+        "password": "The password to connect to MySQL",
+        "host": "The server running MySQL",
+        "port": "The port to connect to MySQL",
+        "database": "The database name to connect to",
+    }
+    deps_pip = ["sqlmodel", "aiomysql"]
+    protocol = "mysql+aiomysql"
+
+    async def create_database(self):
+        from sqlalchemy import text
+        from sqlalchemy.ext.asyncio import create_async_engine
+
+        # Create the engine for the initial connection to the server
+        initial_engine = create_async_engine(self.connection_string().rsplit("/", 1)[0])
+
+        async with initial_engine.connect() as conn:
+            # Check if the database exists
+            result = await conn.execute(text(f"SHOW DATABASES LIKE '{self.database}'"))
+            database_exists = result.scalar() is not None
+
+            # Create the database if it does not exist
+            if not database_exists:
+                # Use aiomysql directly to create the database
+                import aiomysql
+
+                raw_conn = await aiomysql.connect(
+                    user=self.username,
+                    password=self.password,
+                    host=self.host,
+                    port=self.port,
+                )
+                try:
+                    async with raw_conn.cursor() as cursor:
+                        await cursor.execute(f"CREATE DATABASE {self.database}")
+                finally:
+                    await raw_conn.ensure_closed()

bbot/modules/templates/sql.py

@@ -1,3 +1,4 @@
+from contextlib import suppress
 from sqlmodel import SQLModel
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
@@ -88,3 +89,7 @@ def connection_string(self, mask_password=False):
         if self.database:
             connection_string += f"/{self.database}"
         return connection_string
+
+    async def cleanup(self):
+        with suppress(Exception):
+            await self.engine.dispose()

bbot/test/test_step_2/module_tests/test_module_mysql.py

@@ -0,0 +1,76 @@
+import asyncio
+import time
+
+from .base import ModuleTestBase
+
+
+class TestMySQL(ModuleTestBase):
+    targets = ["evilcorp.com"]
+    skip_distro_tests = True
+
+    async def setup_before_prep(self, module_test):
+        process = await asyncio.create_subprocess_exec(
+            "docker",
+            "run",
+            "--name",
+            "bbot-test-mysql",
+            "--rm",
+            "-e",
+            "MYSQL_ROOT_PASSWORD=bbotislife",
+            "-e",
+            "MYSQL_DATABASE=bbot",
+            "-p",
+            "3306:3306",
+            "-d",
+            "mysql",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+
+        import aiomysql
+
+        # wait for the container to start
+        start_time = time.time()
+        while True:
+            try:
+                conn = await aiomysql.connect(user="root", password="bbotislife", db="bbot", host="localhost")
+                conn.close()
+                break
+            except Exception as e:
+                if time.time() - start_time > 60:  # timeout after 60 seconds
+                    self.log.error("MySQL server did not start in time.")
+                    raise e
+                await asyncio.sleep(1)
+
+        if process.returncode != 0:
+            self.log.error(f"Failed to start MySQL server: {stderr.decode()}")
+
+    async def check(self, module_test, events):
+        import aiomysql
+
+        # Connect to the MySQL database
+        conn = await aiomysql.connect(user="root", password="bbotislife", db="bbot", host="localhost")
+
+        try:
+            async with conn.cursor() as cur:
+                await cur.execute("SELECT * FROM event")
+                events = await cur.fetchall()
+                assert len(events) == 3, "No events found in MySQL database"
+
+                await cur.execute("SELECT * FROM scan")
+                scans = await cur.fetchall()
+                assert len(scans) == 1, "No scans found in MySQL database"
+
+                await cur.execute("SELECT * FROM target")
+                targets = await cur.fetchall()
+                assert len(targets) == 1, "No targets found in MySQL database"
+        finally:
+            conn.close()
+            process = await asyncio.create_subprocess_exec(
+                "docker", "stop", "bbot-test-mysql", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                raise Exception(f"Failed to stop MySQL server: {stderr.decode()}")

docs/scanning/output.md

@@ -237,6 +237,26 @@ config:
       password: bbotislife
 ```
 
+### MySQL
+
+The `mysql` output module allows you to ingest events, scans, and targets into a MySQL database. By default, it will connect to the server on `localhost` with a username of `root` and password of `bbotislife`. You can change this behavior in the config.
+
+```bash
+# specifying an alternate database
+bbot -t evilcorp.com -om mysql -c modules.mysql.database=custom_bbot_db
+```
+
+```yaml title="mysql_preset.yml"
+config:
+  modules:
+    mysql:
+      host: mysql.fsociety.local
+      database: custom_bbot_db
+      port: 3306
+      username: root
+      password: bbotislife
+```
+
 ### Subdomains
 
 The `subdomains` output module produces simple text file containing only in-scope and resolved subdomains: