Remove the need for the asyncio stop Event on attacks, refactor attac…

…k module to a single function that can be cancelled on ctrl+c, use a finally block to persist attacked_ids in case of interruption

tests/attack/test_mod_backup.py

@@ -1,4 +1,3 @@
-from asyncio import Event
 from unittest.mock import AsyncMock
 
 import httpx
@@ -35,7 +34,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleBackup(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleBackup(crawler, persister, options, crawler_configuration)
         module.do_get = True
         await module.attack(request, response)
 
@@ -66,6 +65,6 @@ async def test_false_positive():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleBackup(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleBackup(crawler, persister, options, crawler_configuration)
         module.do_get = True
         assert not await module.must_attack(request, response)

tests/attack/test_mod_buster.py

@@ -1,6 +1,6 @@
 from unittest import mock
 from unittest.mock import AsyncMock
-from asyncio import Event, sleep
+from asyncio import sleep
 
 import httpx
 import respx
@@ -41,7 +41,7 @@ async def test_whole_stuff():
             "wordlist.txt": "nawak\nadmin\nconfig.inc\nauthconfig.php",
         }
         with mock.patch("builtins.open", get_mock_open(files)):
-            module = ModuleBuster(crawler, persister, options, Event(), crawler_configuration)
+            module = ModuleBuster(crawler, persister, options, crawler_configuration)
             module.DATA_DIR = ""
             module.PATHS_FILE = "wordlist.txt"
             module.do_get = True

tests/attack/test_mod_cms.py

@@ -1,7 +1,6 @@
 import os
 import sys
 from os.path import join as path_join
-from asyncio import Event
 from unittest.mock import AsyncMock
 
 import httpx
@@ -41,7 +40,7 @@ async def test_no_drupal():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -79,7 +78,7 @@ async def test_drupal_version_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -129,7 +128,7 @@ async def test_drupal_multi_versions_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -175,7 +174,7 @@ async def test_drupal_version_not_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -209,7 +208,7 @@ async def test_no_joomla():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -248,7 +247,7 @@ async def test_joomla_version_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -298,7 +297,7 @@ async def test_joomla_multi_versions_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -345,7 +344,7 @@ async def test_joomla_version_not_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -379,7 +378,7 @@ async def test_no_prestashop():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -420,7 +419,7 @@ async def test_prestashop_version_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -470,7 +469,7 @@ async def test_prestashop_multi_versions_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -515,7 +514,7 @@ async def test_prestashop_version_not_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -558,7 +557,7 @@ async def test_spip_version_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -609,7 +608,7 @@ async def test_spip_multi_versions_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -655,7 +654,7 @@ async def test_spip_version_not_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -698,7 +697,7 @@ async def test_wp_version_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -750,7 +749,7 @@ async def test_wp_multi_versions_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -797,7 +796,7 @@ async def test_wp_no_version_detected():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -892,7 +891,7 @@ async def test_wp_false_positive_403():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -1004,7 +1003,7 @@ async def test_wp_false_positive_success():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -1094,7 +1093,7 @@ async def test_wp_plugin():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 
@@ -1181,7 +1180,7 @@ async def test_wp_theme():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "tasks": 20}
 
-        module = ModuleCms(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCms(crawler, persister, options, crawler_configuration)
 
         await module.attack(request)
 

tests/attack/test_mod_cookieflags.py

@@ -1,4 +1,3 @@
-import asyncio
 import re
 from unittest.mock import AsyncMock
 
@@ -35,7 +34,7 @@ async def test_cookieflags():
         await crawler.async_send(request)  # Put cookies in our crawler object
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleCookieflags(crawler, persister, options, asyncio.Event(), crawler_configuration)
+        module = ModuleCookieflags(crawler, persister, options, crawler_configuration)
         await module.attack(request)
 
         cookie_flags = []

tests/attack/test_mod_crlf.py

@@ -1,4 +1,3 @@
-from asyncio import Event
 from unittest.mock import AsyncMock
 
 import respx
@@ -29,7 +28,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleCrlf(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCrlf(crawler, persister, options, crawler_configuration)
         module.do_get = True
         await module.attack(request)
 

tests/attack/test_mod_csrf.py

@@ -2,7 +2,6 @@
 import os
 import sys
 from time import sleep
-from asyncio import Event
 from unittest.mock import AsyncMock
 
 import httpx
@@ -90,7 +89,7 @@ async def test_csrf_cases():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 1}
 
-        module = ModuleCsrf(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleCsrf(crawler, persister, options, crawler_configuration)
         module.do_post = True
         for request, response in all_requests:
             if await module.must_attack(request, response):

tests/attack/test_mod_exec.py

@@ -2,7 +2,7 @@
 import os
 import sys
 from time import sleep
-from asyncio import Event, sleep as Sleep
+from asyncio import sleep as Sleep
 from unittest.mock import AsyncMock
 
 import pytest
@@ -57,7 +57,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleExec(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleExec(crawler, persister, options, crawler_configuration)
         module.do_post = True
         for request in all_requests:
             await module.attack(request)
@@ -103,7 +103,7 @@ async def test_detection():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 1}
 
-        module = ModuleExec(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleExec(crawler, persister, options, crawler_configuration)
         module.do_post = True
         for request in all_requests:
             await module.attack(request)
@@ -137,7 +137,7 @@ def timeout_callback(http_request):
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 1, "level": 1}
 
-        module = ModuleExec(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleExec(crawler, persister, options, crawler_configuration)
         module.do_post = False
 
         payloads_until_sleep = 0

tests/attack/test_mod_file.py

@@ -2,7 +2,7 @@
 import os
 import sys
 from time import sleep
-from asyncio import Event, sleep as Sleep
+from asyncio import sleep as Sleep
 from unittest.mock import AsyncMock
 
 import httpx
@@ -36,7 +36,7 @@ async def test_inclusion_detection():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleFile(crawler, persister, options, crawler_configuration)
         module.do_post = False
         await module.attack(request)
 
@@ -56,7 +56,7 @@ async def test_open_redirect():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleFile(crawler, persister, options, crawler_configuration)
         module.do_post = False
         await module.attack(request)
 
@@ -74,7 +74,7 @@ async def test_loknop_lfi_to_rce():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleFile(crawler, persister, options, crawler_configuration)
         module.do_post = False
         await module.attack(request)
 
@@ -101,7 +101,7 @@ async def test_warning_false_positive():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleFile(crawler, persister, options, crawler_configuration)
         module.do_post = False
         await module.attack(request)
 
@@ -130,7 +130,7 @@ async def test_no_crash():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleFile(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleFile(crawler, persister, options, crawler_configuration)
         module.do_post = False
         for request in all_requests:
             await module.attack(request)

tests/attack/test_mod_htaccess.py

@@ -1,4 +1,3 @@
-from asyncio import Event
 from unittest.mock import AsyncMock
 
 import httpx
@@ -46,7 +45,7 @@ async def test_whole_stuff():
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2}
 
-        module = ModuleHtaccess(crawler, persister, options, Event(), crawler_configuration)
+        module = ModuleHtaccess(crawler, persister, options, crawler_configuration)
         module.do_get = True
         for request, response in all_requests:
             if await module.must_attack(request, response):