From e110f86f39281eed6edf293b7350218f8eaceac1 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 11 Apr 2023 18:29:12 -0500 Subject: [PATCH 1/6] added extensions and status codes into auto filtering decision calculus --- src/heuristics.rs | 444 +++++++++++++++++++++++---------------- tests/test_heuristics.rs | 63 +++++- 2 files changed, 328 insertions(+), 179 deletions(-) diff --git a/src/heuristics.rs b/src/heuristics.rs index ca7958cf..20d5d6e9 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::collections::HashMap; use anyhow::{bail, Result}; use scraper::{Html, Selector}; @@ -276,133 +277,183 @@ impl HeuristicTests { None }; - // 4 is due to the array in the nested for loop below - let mut responses = Vec::with_capacity(4); + // 6 is due to the array in the nested for loop below + let mut responses = Vec::with_capacity(6); + + let extensions = if self.handles.config.extensions.is_empty() { + vec!["".to_string()] + } else { + let mut exts: Vec = self + .handles + .config + .extensions + .iter() + .map(|ext| format!(".{}", ext)) + .collect(); + exts.push("".to_string()); + exts + }; // for every method, attempt to id its 404 response // // a good example of one where the GET/POST differ is on hackthebox: // - http://prd.m.rendering-api.interface.htb/api + // + // a good example of one where the heuristics return a 403 and a 404 (apache) + // as well as return two different types of 404s based on the file extension + // - http://10.10.11.198 (Encoding box in normal labs) + // + // both methods and extensions can elicit different responses from a given + // server, so both are considered when building auto-filter rules for method in self.handles.config.methods.iter() { - for (prefix, length) in [("", 1), ("", 3), (".htaccess", 1), ("admin", 1)] { - let path = format!("{prefix}{}", self.unique_string(length)); - - let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone()); - - let nonexistent_url = ferox_url.format(&path, slash)?; - - // example requests: - // - http://localhost/2fc1077836ad43ab98b7a31c2ca28fea - // - http://localhost/92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f - // - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09 - // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280 - let response = - logged_request(&nonexistent_url, method, data, self.handles.clone()).await; - - req_counter += 1; - - // continue to next on error - let response = skip_fail!(response); - - if !self - .handles - .config - .status_codes - .contains(&response.status().as_u16()) - { - // if the response code isn't one that's accepted via -s values, then skip to the next - // - // the default value for -s is all status codes, so unless the user says otherwise - // this won't fire - continue; - } + for extension in extensions.iter() { + for (prefix, length) in [ + ("", 1), + ("", 3), + (".htaccess", 1), + (".htaccess", 3), + ("admin", 1), + ("admin", 3), + ] { + let path = format!("{prefix}{}{extension}", self.unique_string(length)); + + let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone()); + + let nonexistent_url = ferox_url.format(&path, slash)?; + + // example requests: + // - http://localhost/2fc1077836ad43ab98b7a31c2ca28fea + // - http://localhost/92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f + // - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09 + // - http://localhost/.htaccess92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f + // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280 + // - http://localhost/admin92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f + let response = + logged_request(&nonexistent_url, method, data, self.handles.clone()).await; + + req_counter += 1; + + // continue to next on error + let response = skip_fail!(response); + + if !self + .handles + .config + .status_codes + .contains(&response.status().as_u16()) + { + // if the response code isn't one that's accepted via -s values, then skip to the next + // + // the default value for -s is all status codes, so unless the user says otherwise + // this won't fire + continue; + } - let ferox_response = FeroxResponse::from( - response, - &ferox_url.target, - method, - self.handles.config.output_level, - ) - .await; + let ferox_response = FeroxResponse::from( + response, + &ferox_url.target, + method, + self.handles.config.output_level, + ) + .await; - responses.push(ferox_response); - } + responses.push(ferox_response); + } - if responses.len() < 2 { - // don't have enough responses to make a determination, continue to next method - responses.clear(); - continue; - } + if responses.len() < 2 { + // don't have enough responses to make a determination, continue to next method + responses.clear(); + continue; + } - // Command::AddFilter, &str (bytes/words/lines), usize (i.e. length associated with the type) - let Some(filter) = self.examine_404_like_responses(&responses) else { - // no match was found during analysis of responses - responses.clear(); - continue; - }; + // check the responses for similarities on which we can filter, multiple may be returned + let Some((wildcard_filters, wildcard_responses)) = self.examine_404_like_responses(&responses) else { + // no match was found during analysis of responses + responses.clear(); + log::warn!("no match found for 404 responses"); + continue; + }; - // report to the user, if appropriate - if matches!( - self.handles.config.output_level, - OutputLevel::Default | OutputLevel::Quiet - ) { - // sentry value to control whether or not to print the filter - // used because we only want to print the same filter once - let mut print_sentry = true; - - if let Ok(filters) = self.handles.filters.data.filters.read() { - for other in filters.iter() { - if let Some(other_wildcard) = - other.as_any().downcast_ref::() - { - if &*filter == other_wildcard { - print_sentry = false; - break; + // report to the user, if appropriate + if matches!( + self.handles.config.output_level, + OutputLevel::Default | OutputLevel::Quiet + ) { + // sentry value to control whether or not to print the filter + // used because we only want to print the same filter once + let mut print_sentry; + + if let Ok(filters) = self.handles.filters.data.filters.read() { + for new_wildcard in &wildcard_filters { + // reset the sentry for every new wildcard produced by examine_404_like_responses + print_sentry = true; + + for other in filters.iter() { + if let Some(other_wildcard) = + other.as_any().downcast_ref::() + { + // check the new wildcard against all existing wildcards, if it was added + // on the cli or by a previous directory, don't print it + if new_wildcard.as_ref() == other_wildcard { + print_sentry = false; + break; + } + } + } + + // if we're here, we've found a new wildcard that we didn't previously display, print it + if print_sentry { + ferox_print(&format!("{}", new_wildcard), &PROGRESS_PRINTER); } } } } - if print_sentry { - ferox_print(&format!("{}", filter), &PROGRESS_PRINTER); + // create the new filter + for wildcard in wildcard_filters { + self.handles.filters.send(Command::AddFilter(wildcard))?; } - } - // create the new filter - self.handles.filters.send(Command::AddFilter(filter))?; + // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line + // + // in addition, we'll create a similarity filter as a fallback + for resp in wildcard_responses { + let hash = SIM_HASHER.create_signature(preprocess(resp.text()).iter()); - // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line - // - // in addition, we'll create a similarity filter as a fallback - let hash = SIM_HASHER.create_signature(preprocess(responses[0].text()).iter()); + let sim_filter = SimilarityFilter { + hash, + original_url: resp.url().to_string(), + }; - let sim_filter = SimilarityFilter { - hash, - original_url: responses[0].url().to_string(), - }; + self.handles + .filters + .send(Command::AddFilter(Box::new(sim_filter)))?; - self.handles - .filters - .send(Command::AddFilter(Box::new(sim_filter)))?; + if resp.is_directory() { + // response is either a 3XX with a Location header that matches url + '/' + // or it's a 2XX that ends with a '/' + // or it's a 403 that ends with a '/' - if responses[0].is_directory() { - // response is either a 3XX with a Location header that matches url + '/' - // or it's a 2XX that ends with a '/' - // or it's a 403 that ends with a '/' + // set the wildcard flag to true, so we can check it when preventing + // recursion in event_handlers/scans.rs - // set the wildcard flag to true, so we can check it when preventing - // recursion in event_handlers/scans.rs - responses[0].set_wildcard(true); + // we'd need to clone the response to give ownership to the global list anyway + // so we'll also use that clone to set the wildcard flag + let mut cloned_resp = resp.clone(); - // add the response to the global list of responses - RESPONSES.insert(responses[0].clone()); + cloned_resp.set_wildcard(true); - // function-internal magic number, indicates that we've detected a wildcard directory - req_counter += 100; - } + // add the response to the global list of responses + RESPONSES.insert(cloned_resp); + + // function-internal magic number, indicates that we've detected a wildcard directory + req_counter += 100; + } + } - // reset the responses for the next method, if it exists - responses.clear(); + // reset the responses for the next method, if it exists + responses.clear(); + } } log::trace!("exit: detect_404_like_responses"); @@ -416,96 +467,137 @@ impl HeuristicTests { Ok(Some(retval)) } - /// for all responses, examine chars/words/lines - /// if all responses respective lengths match each other, we can assume - /// that will remain true for subsequent non-existent urls + /// for all responses, group them by status code, then examine chars/words/lines. + /// if all responses' respective lengths within a status code grouping match + /// each other, we can assume that will remain true for subsequent non-existent urls /// - /// values are examined from most to least specific (content length, word count, line count) - fn examine_404_like_responses( + /// within a status code grouping, values are examined from most to + /// least specific (content length, word count, line count) + fn examine_404_like_responses<'a>( &self, - responses: &[FeroxResponse], - ) -> Option> { + responses: &'a [FeroxResponse], + ) -> Option<(Vec>, Vec<&'a FeroxResponse>)> { + // aside from word/line/byte counts, additional discriminators are status code + // extension, and request method. The request method and extension are handled by + // the caller, since they're part of the request and make up the nested for loops + // in detect_404_like_responses. + // + // The status code is handled here, since it's part of the response to catch cases + // where we have something like a 403 and a 404 + let mut size_sentry = true; let mut word_sentry = true; let mut line_sentry = true; - let method = responses[0].method(); - let status_code = responses[0].status(); - let content_length = responses[0].content_length(); - let word_count = responses[0].word_count(); - let line_count = responses[0].line_count(); - - for response in &responses[1..] { - // if any of the responses differ in length, that particular - // response length type is no longer a candidate for filtering - if response.content_length() != content_length { - size_sentry = false; - } + // returned vec of boxed wildcard filters + let mut wildcards = Vec::new(); - if response.word_count() != word_count { - word_sentry = false; - } + // returned vec of ferox responses that are needed for additional + // analysis + let mut wild_responses = Vec::new(); - if response.line_count() != line_count { - line_sentry = false; - } - } + // mapping of grouped responses to status code + let mut grouped_responses = HashMap::new(); - if !size_sentry && !word_sentry && !line_sentry { - // none of the response lengths match, so we can't filter on any of them - return None; + // iterate over all responses and add each response to its + // corresponding status code group + for response in responses { + grouped_responses + .entry(response.status()) + .or_insert_with(Vec::new) + .push(response); } - let mut wildcard = WildcardFilter { - content_length: None, - line_count: None, - word_count: None, - method: method.to_string(), - status_code: status_code.as_u16(), - dont_filter: self.handles.config.dont_filter, - }; - - match (size_sentry, word_sentry, line_sentry) { - (true, true, true) => { - // all three types of length match, so we can't filter on any of them - wildcard.content_length = Some(content_length); - wildcard.word_count = Some(word_count); - wildcard.line_count = Some(line_count); - } - (true, true, false) => { - // content length and word count match, so we can filter on either - wildcard.content_length = Some(content_length); - wildcard.word_count = Some(word_count); - } - (true, false, true) => { - // content length and line count match, so we can filter on either - wildcard.content_length = Some(content_length); - wildcard.line_count = Some(line_count); - } - (false, true, true) => { - // word count and line count match, so we can filter on either - wildcard.word_count = Some(word_count); - wildcard.line_count = Some(line_count); - } - (true, false, false) => { - // content length matches, so we can filter on that - wildcard.content_length = Some(content_length); - } - (false, true, false) => { - // word count matches, so we can filter on that - wildcard.word_count = Some(word_count); + // iterate over each grouped response and determine the most specific + // filter that can be applied to all responses in the group, i.e. + // start from byte count and work 'out' to line count + for (_, response_group) in &grouped_responses { + if response_group.len() < 2 { + // not enough responses to make a determination + continue; } - (false, false, true) => { - // line count matches, so we can filter on that - wildcard.line_count = Some(line_count); + + let method = response_group[0].method(); + let status_code = response_group[0].status(); + let content_length = response_group[0].content_length(); + let word_count = response_group[0].word_count(); + let line_count = response_group[0].line_count(); + + for response in &response_group[1..] { + // if any of the responses differ in length, that particular + // response length type is no longer a candidate for filtering + if response.content_length() != content_length { + size_sentry = false; + } + + if response.word_count() != word_count { + word_sentry = false; + } + + if response.line_count() != line_count { + line_sentry = false; + } } - (false, false, false) => { - // none of the length types match, so we can't filter on any of them - unreachable!("no wildcard size matches; handled by the if statement above"); + + if !size_sentry && !word_sentry && !line_sentry { + // none of the response lengths match, so we can't filter on any of them + continue; } - }; - Some(Box::new(wildcard)) + let mut wildcard = WildcardFilter { + content_length: None, + line_count: None, + word_count: None, + method: method.to_string(), + status_code: status_code.as_u16(), + dont_filter: self.handles.config.dont_filter, + }; + + match (size_sentry, word_sentry, line_sentry) { + (true, true, true) => { + // all three types of length match, so we can't filter on any of them + wildcard.content_length = Some(content_length); + wildcard.word_count = Some(word_count); + wildcard.line_count = Some(line_count); + } + (true, true, false) => { + // content length and word count match, so we can filter on either + wildcard.content_length = Some(content_length); + wildcard.word_count = Some(word_count); + } + (true, false, true) => { + // content length and line count match, so we can filter on either + wildcard.content_length = Some(content_length); + wildcard.line_count = Some(line_count); + } + (false, true, true) => { + // word count and line count match, so we can filter on either + wildcard.word_count = Some(word_count); + wildcard.line_count = Some(line_count); + } + (true, false, false) => { + // content length matches, so we can filter on that + wildcard.content_length = Some(content_length); + } + (false, true, false) => { + // word count matches, so we can filter on that + wildcard.word_count = Some(word_count); + } + (false, false, true) => { + // line count matches, so we can filter on that + wildcard.line_count = Some(line_count); + } + (false, false, false) => { + // none of the length types match, so we can't filter on any of them + unreachable!("no wildcard size matches; handled by the if statement above"); + } + }; + + wild_responses.push(response_group[0]); + wildcards.push(Box::new(wildcard)); + } + + Some((wildcards, wild_responses)) } } diff --git a/tests/test_heuristics.rs b/tests/test_heuristics.rs index 6696a033..c86df673 100644 --- a/tests/test_heuristics.rs +++ b/tests/test_heuristics.rs @@ -164,7 +164,7 @@ fn test_static_wildcard_request_found() -> Result<(), Box let mock = srv.mock(|when, then| { when.method(GET) - .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap()); + .path_matches(Regex::new("/[.a-zA-Z0-9]{32,}/").unwrap()); then.status(200).body("this is a test"); }); @@ -188,7 +188,8 @@ fn test_static_wildcard_request_found() -> Result<(), Box .and(predicate::str::contains("1l")), ); - assert_eq!(mock.hits(), 1); + assert_eq!(mock.hits(), 6); + Ok(()) } @@ -305,11 +306,67 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled( .success() .stdout(predicate::str::contains(srv.url("/"))); - assert_eq!(mock.hits(), 4); + assert_eq!(mock.hits(), 6); assert_eq!(mock2.hits(), 1); Ok(()) } +#[test] +/// test finds a 404-like response that returns a 403 and a 403 directory should still be allowed +/// to be tested for recrusion +fn heuristics_wildcard_test_that_auto_filtering_403s_still_allows_for_recursion_into_403_directories( +) -> Result<(), Box> { + let srv = MockServer::start(); + + let super_long = String::from("92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f09d471004154b83bcc1c6ec49f6f"); + + let (tmp_dir, file) = + setup_tmp_directory(&["LICENSE".to_string(), super_long.clone()], "wordlist")?; + + srv.mock(|when, then| { + when.method(GET) + .path_matches(Regex::new("/.?[a-zA-Z0-9]{32,103}").unwrap()); + then.status(403) + .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + }); + + srv.mock(|when, then| { + when.method(GET).path("/LICENSE/"); + then.status(403) + .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + }); + + srv.mock(|when, then| { + when.method(GET).path(format!("/LICENSE/{}", super_long)); + then.status(200); + }); + + let cmd = Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg(srv.url("/")) + .arg("--wordlist") + .arg(file.as_os_str()) + .arg("--add-slash") + .unwrap(); + + teardown_tmp_directory(tmp_dir); + + cmd.assert().success().stdout( + predicate::str::contains("GET") + .and(predicate::str::contains( + "Auto-filtering found 404-like response and created new filter", + )) + .and(predicate::str::contains("403")) + .and(predicate::str::contains("1l")) + .and(predicate::str::contains("4w")) + .and(predicate::str::contains("46c")) + .and(predicate::str::contains(srv.url("/LICENSE/LICENSE/"))), + ); + + Ok(()) +} + // #[test] // /// test finds a static wildcard and reports as much to stdout and a file // fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() { From b375893461270b14777f1f45bc7c8137e4ea8db9 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 11 Apr 2023 18:32:56 -0500 Subject: [PATCH 2/6] nitpickery --- src/heuristics.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/heuristics.rs b/src/heuristics.rs index 20d5d6e9..09e35f76 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -1,5 +1,5 @@ -use std::sync::Arc; use std::collections::HashMap; +use std::sync::Arc; use anyhow::{bail, Result}; use scraper::{Html, Selector}; @@ -280,19 +280,14 @@ impl HeuristicTests { // 6 is due to the array in the nested for loop below let mut responses = Vec::with_capacity(6); - let extensions = if self.handles.config.extensions.is_empty() { - vec!["".to_string()] - } else { - let mut exts: Vec = self - .handles - .config - .extensions - .iter() - .map(|ext| format!(".{}", ext)) - .collect(); - exts.push("".to_string()); - exts - }; + // no matter what, we want an empty extension for the base case + let mut extensions = vec!["".to_string()]; + + // and then we want to add any extensions that was specified + // or has since been added to the running config + for ext in &self.handles.config.extensions { + extensions.push(format!(".{}", ext)); + } // for every method, attempt to id its 404 response // From 4f679592b86dd759aa6e118517ec8e0b3925a221 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 11 Apr 2023 18:34:02 -0500 Subject: [PATCH 3/6] bumped version to 2.9.3 --- Cargo.lock | 2 +- Cargo.toml | 9 +++++++-- shell_completions/_feroxbuster | 4 ++-- shell_completions/_feroxbuster.ps1 | 4 ++-- shell_completions/feroxbuster.elv | 4 ++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6baaca7..1b121b9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,7 +782,7 @@ dependencies = [ [[package]] name = "feroxbuster" -version = "2.9.2" +version = "2.9.3" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 7cd3cc59..83344aff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "2.9.2" +version = "2.9.3" authors = ["Ben 'epi' Risher (@epi052)"] license = "MIT" edition = "2021" @@ -56,7 +56,12 @@ ctrlc = "3.2.2" anyhow = "1.0.69" leaky-bucket = "0.12.1" gaoya = "0.1.2" -self_update = {version = "0.36.0", features = ["archive-tar", "compression-flate2", "archive-zip", "compression-zip-deflate"]} +self_update = { version = "0.36.0", features = [ + "archive-tar", + "compression-flate2", + "archive-zip", + "compression-zip-deflate", +] } [dev-dependencies] tempfile = "3.3.0" diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index 04c7d580..1f46462d 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -24,8 +24,8 @@ _feroxbuster() { '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \ '*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \ '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \ -'-a+[Sets the User-Agent (default: feroxbuster/2.9.2)]:USER_AGENT: ' \ -'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.2)]:USER_AGENT: ' \ +'-a+[Sets the User-Agent (default: feroxbuster/2.9.3)]:USER_AGENT: ' \ +'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.3)]:USER_AGENT: ' \ '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \ '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \ '*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \ diff --git a/shell_completions/_feroxbuster.ps1 b/shell_completions/_feroxbuster.ps1 index aa141b2c..6972cde8 100644 --- a/shell_completions/_feroxbuster.ps1 +++ b/shell_completions/_feroxbuster.ps1 @@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock { [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests') [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)') [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)') - [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.2)') - [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.2)') + [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.3)') + [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.3)') [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)') [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)') [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)') diff --git a/shell_completions/feroxbuster.elv b/shell_completions/feroxbuster.elv index c0594e2f..a4d6c4f2 100644 --- a/shell_completions/feroxbuster.elv +++ b/shell_completions/feroxbuster.elv @@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words| cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests' cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)' cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)' - cand -a 'Sets the User-Agent (default: feroxbuster/2.9.2)' - cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.2)' + cand -a 'Sets the User-Agent (default: feroxbuster/2.9.3)' + cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.3)' cand -x 'File extension(s) to search for (ex: -x php -x pdf js)' cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)' cand -m 'Which HTTP request method(s) should be sent (default: GET)' From ea81600850c45e6a65094150452de96e94aa28e9 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 11 Apr 2023 18:36:37 -0500 Subject: [PATCH 4/6] clippy --- src/heuristics.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/heuristics.rs b/src/heuristics.rs index 09e35f76..6b51a08f 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -468,6 +468,7 @@ impl HeuristicTests { /// /// within a status code grouping, values are examined from most to /// least specific (content length, word count, line count) + #[allow(clippy::vec_box)] // the box is needed in the caller and i dont feel like changing it fn examine_404_like_responses<'a>( &self, responses: &'a [FeroxResponse], @@ -506,7 +507,7 @@ impl HeuristicTests { // iterate over each grouped response and determine the most specific // filter that can be applied to all responses in the group, i.e. // start from byte count and work 'out' to line count - for (_, response_group) in &grouped_responses { + for response_group in grouped_responses.values() { if response_group.len() < 2 { // not enough responses to make a determination continue; From b5472f5341ff320d45503fc479a2170f4b98c414 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 11 Apr 2023 18:39:28 -0500 Subject: [PATCH 5/6] updated deps --- Cargo.lock | 31 ++++++++++++++++++++++--------- Cargo.toml | 6 +++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b121b9f..6532e8fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,6 +19,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "getrandom 0.2.8", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -635,9 +647,9 @@ dependencies = [ [[package]] name = "dirs" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "dece029acd3353e3a58ac2e3eb3c8d6c35827a892edc6cc4138ef9c33df46ecd" dependencies = [ "dirs-sys", ] @@ -654,13 +666,13 @@ dependencies = [ [[package]] name = "dirs-sys" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "04414300db88f70d74c5ff54e50f9e1d1737d9a5b90f53fcf2e95ca2a9ab554b" dependencies = [ "libc", "redox_users", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -1020,7 +1032,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f5a92848d5984b9e3cea74c8df667ffb79ee6f181e2cf9b0df2e50c2f96cabb" dependencies = [ - "ahash", + "ahash 0.7.6", "crossbeam-utils", "fnv", "itertools", @@ -2317,15 +2329,16 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scraper" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c557a9a03db98b0b298b497f0e16cd35a04a1fa9ee1130a6889c0714e0b73df" +checksum = "59e25654b5e9fd557a67dbaab5a5d36b8c448d0561beb4c041b6dbb902eddfa6" dependencies = [ + "ahash 0.8.3", "cssparser", "ego-tree", "getopts", "html5ever", - "matches", + "once_cell", "selectors", "smallvec", "tendril", diff --git a/Cargo.toml b/Cargo.toml index 83344aff..1b3e1e67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,10 +26,10 @@ clap = { version = "4.1.8", features = ["wrap_help", "cargo"] } clap_complete = "4.1.4" regex = "1.5.5" lazy_static = "1.4.0" -dirs = "4.0.0" +dirs = "5.0.0" [dependencies] -scraper = "0.15.0" +scraper = "0.16.0" futures = "0.3.26" tokio = { version = "1.26.0", features = ["full"] } tokio-util = { version = "0.7.7", features = ["codec"] } @@ -48,7 +48,7 @@ uuid = { version = "1.3.0", features = ["v4"] } indicatif = "0.15" console = "0.15.2" openssl = { version = "0.10", features = ["vendored"] } -dirs = "4.0.0" +dirs = "5.0.0" regex = "1.5.5" crossterm = "0.26.0" rlimit = "0.9.1" From 1ced3b5d77997407a95f09a887f68b04c0d8bdd3 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 11 Apr 2023 18:48:18 -0500 Subject: [PATCH 6/6] modified msg when dir listing is found with dont-extract --- src/scanner/ferox_scanner.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/scanner/ferox_scanner.rs b/src/scanner/ferox_scanner.rs index a7f61c36..c117a841 100644 --- a/src/scanner/ferox_scanner.rs +++ b/src/scanner/ferox_scanner.rs @@ -283,7 +283,11 @@ impl FeroxScanner { let mut message = format!("=> {}", style("Directory listing").blue().bright()); if !self.handles.config.extract_links { - write!(message, " (add {} to scan)", style("-e").bright().yellow())?; + write!( + message, + " (remove {} to scan)", + style("--dont-extract-links").bright().yellow() + )?; } if !self.handles.config.force_recursion {