Headless crawler

Pipfile

@@ -30,6 +30,9 @@ sslyze = "==5.0.1"
 humanize = "==3.13.1"
 mitmproxy = "==8.0.0"
 h11 = "==0.12"
+arsenic = "==21.8"
+pyasn1 = "==0.4.8"
+aiohttp = "==3.8.1"
 
 [requires]
 python_version = "3.8"

setup.py

@@ -117,12 +117,15 @@ def run_tests(self):
         "sqlalchemy>=1.4.26",
         "aiocache==0.11.1",
         "aiosqlite==0.17.0",
+        "aiohttp==3.8.1",
         "loguru>=0.5.3",
         "dnspython==2.1.0",
         "httpcore>=0.15.0",
         "mitmproxy==8.0.0",
         "h11==0.12",
-        "pyasn1==0.4.8"
+        "pyasn1==0.4.8",
+        "arsenic==21.8",
+        "pyasn1==0.4.8",
     ],
     extras_require={
         "NTLM": ["httpx-ntlm"],

tests/__init__.py

@@ -1,11 +1,3 @@
-from unittest.mock import MagicMock
-
-
-class AsyncMock(MagicMock):
-    async def __call__(self, *args, **kwargs):
-        return super(AsyncMock, self).__call__(*args, **kwargs)
-
-
 class AsyncIterator:
     def __init__(self, seq):
         self.iter = iter(seq)

tests/attack/test_mod_backup.py

@@ -1,4 +1,5 @@
 from asyncio import Event
+from unittest.mock import AsyncMock
 
 import httpx
 import respx
@@ -8,7 +9,6 @@
 from wapitiCore.net.crawler import AsyncCrawler
 from wapitiCore.net.crawler_configuration import CrawlerConfiguration
 from wapitiCore.attack.mod_backup import ModuleBackup
-from tests import AsyncMock
 
 
 @pytest.mark.asyncio
@@ -35,7 +35,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleBackup(crawler, persister, options, Event())
+        module = ModuleBackup(crawler, persister, options, Event(), crawler_configuration)
         module.do_get = True
         await module.attack(request, response)
 
@@ -66,6 +66,6 @@ async def test_false_positive():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleBackup(crawler, persister, options, Event())
+        module = ModuleBackup(crawler, persister, options, Event(), crawler_configuration)
         module.do_get = True
         assert not await module.must_attack(request, response)

tests/attack/test_mod_buster.py

@@ -41,7 +41,7 @@ async def test_whole_stuff():
                 "wapitiCore.attack.mod_buster.ModuleBuster.payloads",
                 [("nawak", Flags()), ("admin", Flags()), ("config.inc", Flags()), ("authconfig.php", Flags())]
         ):
-            module = ModuleBuster(crawler, persister, options, Event())
+            module = ModuleBuster(crawler, persister, options, Event(), crawler_configuration)
             module.do_get = True
             await module.attack(request, None)
 

tests/attack/test_mod_cookieflags.py

@@ -1,5 +1,6 @@
 import asyncio
 import re
+from unittest.mock import AsyncMock
 
 import httpx
 import respx
@@ -9,7 +10,6 @@
 from wapitiCore.net.crawler import AsyncCrawler
 from wapitiCore.net.crawler_configuration import CrawlerConfiguration
 from wapitiCore.attack.mod_cookieflags import ModuleCookieflags
-from tests import AsyncMock
 
 
 @pytest.mark.asyncio
@@ -35,7 +35,7 @@ async def test_cookieflags():
         await crawler.async_send(request)  # Put cookies in our crawler object
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleCookieflags(crawler, persister, options, asyncio.Event())
+        module = ModuleCookieflags(crawler, persister, options, asyncio.Event(), crawler_configuration)
         await module.attack(request)
 
         cookie_flags = []

tests/attack/test_mod_crlf.py

@@ -1,4 +1,5 @@
 from asyncio import Event
+from unittest.mock import AsyncMock
 
 import respx
 import pytest
@@ -9,7 +10,6 @@
 from wapitiCore.net.crawler_configuration import CrawlerConfiguration
 from wapitiCore.language.vulnerability import _
 from wapitiCore.attack.mod_crlf import ModuleCrlf
-from tests import AsyncMock
 
 
 @pytest.mark.asyncio
@@ -30,7 +30,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleCrlf(crawler, persister, options, Event())
+        module = ModuleCrlf(crawler, persister, options, Event(), crawler_configuration)
         module.do_get = True
         await module.attack(request)
 

tests/attack/test_mod_csrf.py

@@ -3,6 +3,7 @@
 import sys
 from time import sleep
 from asyncio import Event
+from unittest.mock import AsyncMock
 
 import httpx
 import pytest
@@ -12,7 +13,6 @@
 from wapitiCore.net.crawler_configuration import CrawlerConfiguration
 from wapitiCore.attack.mod_csrf import ModuleCsrf
 from wapitiCore.language.vulnerability import _
-from tests import AsyncMock
 
 
 @pytest.fixture(autouse=True)
@@ -68,7 +68,7 @@ async def test_csrf_cases():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 1}
 
-        module = ModuleCsrf(crawler, persister, options, Event())
+        module = ModuleCsrf(crawler, persister, options, Event(), crawler_configuration)
         module.do_post = True
         for request, response in all_requests:
             if await module.must_attack(request, response):

tests/attack/test_mod_drupal_enum.py

@@ -2,6 +2,7 @@
 import sys
 from os.path import join as path_join
 from asyncio import Event
+from unittest.mock import AsyncMock
 
 import httpx
 import respx
@@ -12,7 +13,6 @@
 from wapitiCore.net.crawler import AsyncCrawler
 from wapitiCore.attack.mod_drupal_enum import ModuleDrupalEnum
 from wapitiCore.language.vulnerability import _
-from tests import AsyncMock
 
 
 # Test no Drupal detected
@@ -42,7 +42,7 @@ async def test_no_drupal():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleDrupalEnum(crawler, persister, options, Event())
+        module = ModuleDrupalEnum(crawler, persister, options, Event(), crawler_configuration)
 
         await module.attack(request)
 
@@ -61,7 +61,11 @@ async def test_version_detected():
         data = changelog.read()
 
     # Response to tell that Drupal is used
-    respx.get("http://perdu.com/core/misc/drupal.js").mock(return_value=httpx.Response(200, headers={"Content-Type": "application/javascript"}))
+    respx.get("http://perdu.com/core/misc/drupal.js").mock(
+        return_value=httpx.Response(
+            200,
+            headers={"Content-Type": "application/javascript"})
+    )
 
     # Response for changelog.txt
     respx.get("http://perdu.com/CHANGELOG.txt").mock(return_value=httpx.Response(200, text=data))
@@ -77,7 +81,7 @@ async def test_version_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleDrupalEnum(crawler, persister, options, Event())
+        module = ModuleDrupalEnum(crawler, persister, options, Event(), crawler_configuration)
 
         await module.attack(request)
 
@@ -106,7 +110,12 @@ async def test_multi_versions_detected():
         data = maintainers.read()
 
     # Response to tell that Drupal is used
-    respx.get("http://perdu.com/core/misc/drupal.js").mock(return_value=httpx.Response(200, headers={"Content-Type": "application/javascript"}))
+    respx.get("http://perdu.com/core/misc/drupal.js").mock(
+        return_value=httpx.Response(
+            200,
+            headers={"Content-Type": "application/javascript"}
+        )
+    )
 
     # Response for  maintainers.txt
     respx.get("http://perdu.com/core/MAINTAINERS.txt").mock(return_value=httpx.Response(200, text=data))
@@ -122,7 +131,7 @@ async def test_multi_versions_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleDrupalEnum(crawler, persister, options, Event())
+        module = ModuleDrupalEnum(crawler, persister, options, Event(), crawler_configuration)
 
         await module.attack(request)
 
@@ -147,7 +156,12 @@ async def test_version_not_detected():
         data = changelog.read()
 
     # Response to tell that Drupal is used
-    respx.get("http://perdu.com/misc/drupal.js").mock(return_value=httpx.Response(200, headers={"Content-Type": "application/javascript"}))
+    respx.get("http://perdu.com/misc/drupal.js").mock(
+        return_value=httpx.Response(
+            200,
+            headers={"Content-Type": "application/javascript"}
+        )
+    )
 
     # Response for edited changelog.txt
     respx.get("http://perdu.com/CHANGELOG.txt").mock(return_value=httpx.Response(200, text=data))
@@ -163,7 +177,7 @@ async def test_version_not_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleDrupalEnum(crawler, persister, options, Event())
+        module = ModuleDrupalEnum(crawler, persister, options, Event(), crawler_configuration)
 
         await module.attack(request)
 

tests/attack/test_mod_exec.py

@@ -3,6 +3,7 @@
 import sys
 from time import sleep
 from asyncio import Event
+from unittest.mock import AsyncMock
 
 import pytest
 import respx
@@ -13,7 +14,6 @@
 from wapitiCore.language.vulnerability import _
 from wapitiCore.net.crawler import AsyncCrawler
 from wapitiCore.attack.mod_exec import ModuleExec
-from tests import AsyncMock
 
 
 @pytest.fixture(autouse=True)
@@ -57,7 +57,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleExec(crawler, persister, options, Event())
+        module = ModuleExec(crawler, persister, options, Event(), crawler_configuration)
         module.do_post = True
         for request in all_requests:
             await module.attack(request)
@@ -85,7 +85,7 @@ async def test_detection():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 1}
 
-        module = ModuleExec(crawler, persister, options, Event())
+        module = ModuleExec(crawler, persister, options, Event(), crawler_configuration)
         await module.attack(request)
 
         assert persister.add_payload.call_count == 1
@@ -114,7 +114,7 @@ def timeout_callback(http_request):
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 1, "level": 1}
 
-        module = ModuleExec(crawler, persister, options, Event())
+        module = ModuleExec(crawler, persister, options, Event(), crawler_configuration)
         module.do_post = False
 
         payloads_until_sleep = 0

tests/attack/test_mod_file.py

@@ -3,6 +3,7 @@
 import sys
 from time import sleep
 from asyncio import Event
+from unittest.mock import AsyncMock
 
 import pytest
 
@@ -11,7 +12,6 @@
 from wapitiCore.language.vulnerability import _
 from wapitiCore.net.crawler import AsyncCrawler
 from wapitiCore.attack.mod_file import ModuleFile, has_prefix_or_suffix, find_warning_message, FileWarning
-from tests import AsyncMock
 
 
 @pytest.fixture(autouse=True)
@@ -36,7 +36,7 @@ async def test_inclusion_detection():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event())
+        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
         module.do_post = False
         await module.attack(request)
 
@@ -56,7 +56,7 @@ async def test_warning_false_positive():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event())
+        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
         module.do_post = False
         await module.attack(request)
 
@@ -85,7 +85,7 @@ async def test_no_crash():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event())
+        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
         module.do_post = False
         for request in all_requests:
             await module.attack(request)

tests/attack/test_mod_htaccess.py

@@ -1,4 +1,5 @@
 from asyncio import Event
+from unittest.mock import AsyncMock
 
 import httpx
 import respx
@@ -9,7 +10,6 @@
 from wapitiCore.net.crawler import AsyncCrawler
 from wapitiCore.language.vulnerability import _
 from wapitiCore.attack.mod_htaccess import ModuleHtaccess
-from tests import AsyncMock
 
 
 @pytest.mark.asyncio
@@ -47,7 +47,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleHtaccess(crawler, persister, options, Event())
+        module = ModuleHtaccess(crawler, persister, options, Event(), crawler_configuration)
         module.do_get = True
         for request, response in all_requests:
             if await module.must_attack(request, response):