put back use of HTTP redirection urls + raise usage error is auth-typ…

…e=post is used without -s

wapitiCore/main/wapiti.py

@@ -1016,7 +1016,6 @@ async def wapiti_main():
         if args.drop_set_cookie:
             wap.set_drop_cookies()
 
-        auth_credentials = tuple()
         if "credentials" in args:
             if "auth_type" not in args:
                 raise InvalidOptionValue("--auth-type", "This option is required when -a is used")
@@ -1030,8 +1029,12 @@ async def wapiti_main():
         if "auth_type" in args:
             if "credentials" not in args:
                 raise InvalidOptionValue("-a", "This option is required when --auth-type is used")
-            if args.auth_type == "post" and args.starting_urls != []:
+
+            if args.auth_type == "post":
+                if not args.starting_urls:
+                    raise InvalidOptionValue("-s", "This option is required when --auth-type of type 'post' is used")
                 auth_url = args.starting_urls[0]
+
             wap.set_auth_type(args.auth_type)
 
         for bad_param in args.excluded_parameters:

wapitiCore/net/explorer.py

@@ -214,6 +214,9 @@ def extract_links(self, response: Response, request) -> List:
 
         new_requests = []
 
+        if response.is_redirect and self._scope.check(response.redirection_url):
+            allowed_links.append(response.redirection_url)
+
         if "application/x-shockwave-flash" in response.type or request.file_ext == "swf":
             try:
                 swf_links = swf.extract_links_from_swf(response.bytes)

wapitiCore/net/intercepting_explorer.py

@@ -145,6 +145,8 @@ async def response(self, flow):
         content_type = flow.response.headers.get("Content-Type", "text/plain").split(";")[0]
         flow.response.stream = False
 
+        # We only need this for the automated part, a human without the headless crawler will be able to deal
+        # with download popups. Let's keep in mind we should tweak that later.
         is_forced_download = flow.response.headers.get("content-disposition", "").startswith("attachment")
         if not is_interpreted_type(content_type) or is_forced_download:
             flow.response.status_code = 200

wapitiCore/parsers/html.py

@@ -149,8 +149,6 @@ def _cleanup_fragment(self, url: str) -> str:
     @not_empty
     def _iter_raw_links(self) -> Iterator[str]:
         """Generator returning all raw URLs found in HTML "a href", frame's src tags and redirections."""
-        # yield self.redirection_url
-
         for tag in self.soup.find_all("a", href=True):
             yield self._cleanup_fragment(tag["href"]).strip()
 
@@ -395,8 +393,6 @@ def html_redirections(self) -> List[str]:
     @property
     def all_redirections(self) -> Set[str]:
         result = set()
-        # if self.redirection_url:
-        #     result.add(self.redirection_url)
         result.update(self.js_redirections)
         result.update(self.html_redirections)
         return result